2024-11-12 14:16:23,392 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f 2024-11-12 14:16:23,414 main DEBUG Took 0.019355 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-12 14:16:23,414 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-12 14:16:23,414 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-12 14:16:23,416 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-12 14:16:23,417 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:16:23,427 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-12 14:16:23,443 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:16:23,445 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:16:23,446 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:16:23,446 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:16:23,447 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:16:23,447 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:16:23,449 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:16:23,449 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:16:23,450 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:16:23,450 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:16:23,451 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:16:23,451 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:16:23,452 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:16:23,453 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:16:23,453 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:16:23,454 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:16:23,454 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:16:23,455 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:16:23,455 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:16:23,456 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:16:23,456 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:16:23,457 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:16:23,458 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:16:23,458 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-12 14:16:23,459 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:16:23,460 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-12 14:16:23,461 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-12 14:16:23,463 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-12 14:16:23,466 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-12 14:16:23,467 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-12 14:16:23,468 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-12 14:16:23,469 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-12 14:16:23,479 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-12 14:16:23,482 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-12 14:16:23,483 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-12 14:16:23,484 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-12 14:16:23,484 main DEBUG createAppenders(={Console}) 2024-11-12 14:16:23,485 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f initialized 2024-11-12 14:16:23,485 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f 2024-11-12 14:16:23,485 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@7139992f OK. 2024-11-12 14:16:23,486 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-12 14:16:23,486 main DEBUG OutputStream closed 2024-11-12 14:16:23,486 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-12 14:16:23,487 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-12 14:16:23,487 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@4b0d79fc OK 2024-11-12 14:16:23,558 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-12 14:16:23,560 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-12 14:16:23,561 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-12 14:16:23,562 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-12 14:16:23,562 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-12 14:16:23,563 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-12 14:16:23,563 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-12 14:16:23,563 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-12 14:16:23,564 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-12 14:16:23,564 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-12 14:16:23,564 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-12 14:16:23,564 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-12 14:16:23,565 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-12 14:16:23,565 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-12 14:16:23,565 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-12 14:16:23,566 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-12 14:16:23,566 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-12 14:16:23,567 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-12 14:16:23,569 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12 14:16:23,569 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-logging/target/hbase-logging-3.0.0-beta-2-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@68b32e3e) with optional ClassLoader: null 2024-11-12 14:16:23,570 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-12 14:16:23,570 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@68b32e3e] started OK. 2024-11-12T14:16:23,838 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158 2024-11-12 14:16:23,841 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-12 14:16:23,841 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-12T14:16:23,849 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithEagerPolicy timeout: 13 mins 2024-11-12T14:16:23,868 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-12T14:16:23,887 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/cluster_a8bb44ce-6e49-fcce-04c9-975e63e4e40e, deleteOnExit=true 2024-11-12T14:16:23,887 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-12T14:16:23,889 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/test.cache.data in system properties and HBase conf 2024-11-12T14:16:23,889 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/hadoop.tmp.dir in system properties and HBase conf 2024-11-12T14:16:23,890 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/hadoop.log.dir in system properties and HBase conf 2024-11-12T14:16:23,891 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-12T14:16:23,892 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-12T14:16:23,892 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-12T14:16:23,992 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-12T14:16:24,145 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-12T14:16:24,153 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-12T14:16:24,154 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-12T14:16:24,157 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-12T14:16:24,158 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T14:16:24,159 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-12T14:16:24,160 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-12T14:16:24,161 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-12T14:16:24,162 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T14:16:24,163 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-12T14:16:24,163 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/nfs.dump.dir in system properties and HBase conf 2024-11-12T14:16:24,164 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/java.io.tmpdir in system properties and HBase conf 2024-11-12T14:16:24,165 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-12T14:16:24,165 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-12T14:16:24,166 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-12T14:16:25,370 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-12T14:16:25,446 INFO [Time-limited test {}] log.Log(170): Logging initialized @3022ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-12T14:16:25,515 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T14:16:25,569 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T14:16:25,590 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T14:16:25,591 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T14:16:25,592 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-12T14:16:25,605 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T14:16:25,608 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@71c9bba7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/hadoop.log.dir/,AVAILABLE} 2024-11-12T14:16:25,609 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@d942d39{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T14:16:25,829 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@322e478f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/java.io.tmpdir/jetty-localhost-44849-hadoop-hdfs-3_4_1-tests_jar-_-any-11294632649953666131/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-12T14:16:25,836 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@4b008a6a{HTTP/1.1, (http/1.1)}{localhost:44849} 2024-11-12T14:16:25,836 INFO [Time-limited test {}] server.Server(415): Started @3413ms 2024-11-12T14:16:26,786 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-12T14:16:26,808 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-12T14:16:26,818 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-12T14:16:26,818 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-12T14:16:26,818 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-12T14:16:26,820 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@183dc930{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/hadoop.log.dir/,AVAILABLE} 2024-11-12T14:16:26,821 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@288b0180{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-12T14:16:26,965 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@56f5dfd8{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/java.io.tmpdir/jetty-localhost-33791-hadoop-hdfs-3_4_1-tests_jar-_-any-17433523936632536200/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T14:16:26,966 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@5f3319b1{HTTP/1.1, (http/1.1)}{localhost:33791} 2024-11-12T14:16:26,967 INFO [Time-limited test {}] server.Server(415): Started @4544ms 2024-11-12T14:16:27,038 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-12T14:16:28,743 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/cluster_a8bb44ce-6e49-fcce-04c9-975e63e4e40e/data/data2/current/BP-1636810488-172.17.0.3-1731420984850/current, will proceed with Du for space computation calculation, 2024-11-12T14:16:28,743 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/cluster_a8bb44ce-6e49-fcce-04c9-975e63e4e40e/data/data1/current/BP-1636810488-172.17.0.3-1731420984850/current, will proceed with Du for space computation calculation, 2024-11-12T14:16:28,774 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-12T14:16:28,830 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xda6b849de02467c4 with lease ID 0x7371fc716672fecb: Processing first storage report for DS-7eacfa33-eb6e-4d58-855b-667b624c8b17 from datanode DatanodeRegistration(127.0.0.1:42533, datanodeUuid=81b1dfbb-fc6a-4a67-8a95-69c2120de4e8, infoPort=40953, infoSecurePort=0, ipcPort=44687, storageInfo=lv=-57;cid=testClusterID;nsid=2103015766;c=1731420984850) 2024-11-12T14:16:28,831 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xda6b849de02467c4 with lease ID 0x7371fc716672fecb: from storage DS-7eacfa33-eb6e-4d58-855b-667b624c8b17 node DatanodeRegistration(127.0.0.1:42533, datanodeUuid=81b1dfbb-fc6a-4a67-8a95-69c2120de4e8, infoPort=40953, infoSecurePort=0, ipcPort=44687, storageInfo=lv=-57;cid=testClusterID;nsid=2103015766;c=1731420984850), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-12T14:16:28,832 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xda6b849de02467c4 with lease ID 0x7371fc716672fecb: Processing first storage report for DS-b4e3dd8b-84ae-45f5-9d7a-3d1922333865 from datanode DatanodeRegistration(127.0.0.1:42533, datanodeUuid=81b1dfbb-fc6a-4a67-8a95-69c2120de4e8, infoPort=40953, infoSecurePort=0, ipcPort=44687, storageInfo=lv=-57;cid=testClusterID;nsid=2103015766;c=1731420984850) 2024-11-12T14:16:28,832 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xda6b849de02467c4 with lease ID 0x7371fc716672fecb: from storage DS-b4e3dd8b-84ae-45f5-9d7a-3d1922333865 node DatanodeRegistration(127.0.0.1:42533, datanodeUuid=81b1dfbb-fc6a-4a67-8a95-69c2120de4e8, infoPort=40953, infoSecurePort=0, ipcPort=44687, storageInfo=lv=-57;cid=testClusterID;nsid=2103015766;c=1731420984850), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-12T14:16:28,924 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158 2024-11-12T14:16:29,018 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/cluster_a8bb44ce-6e49-fcce-04c9-975e63e4e40e/zookeeper_0, clientPort=50547, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/cluster_a8bb44ce-6e49-fcce-04c9-975e63e4e40e/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/cluster_a8bb44ce-6e49-fcce-04c9-975e63e4e40e/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-12T14:16:29,029 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=50547 2024-11-12T14:16:29,039 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T14:16:29,043 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T14:16:29,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741825_1001 (size=7) 2024-11-12T14:16:29,729 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3 with version=8 2024-11-12T14:16:29,729 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/hbase-staging 2024-11-12T14:16:29,854 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-12T14:16:30,113 INFO [Time-limited test {}] client.ConnectionUtils(128): master/2b6d221c5cde:0 server-side Connection retries=45 2024-11-12T14:16:30,122 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:16:30,123 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T14:16:30,127 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T14:16:30,127 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:16:30,127 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T14:16:30,279 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-12T14:16:30,339 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-12T14:16:30,349 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-12T14:16:30,353 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T14:16:30,383 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 8938 (auto-detected) 2024-11-12T14:16:30,385 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-12T14:16:30,421 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:45591 2024-11-12T14:16:30,458 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:45591 connecting to ZooKeeper ensemble=127.0.0.1:50547 2024-11-12T14:16:30,570 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:455910x0, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T14:16:30,576 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:45591-0x1012f67a1e60000 connected 2024-11-12T14:16:30,674 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T14:16:30,677 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T14:16:30,689 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45591-0x1012f67a1e60000, quorum=127.0.0.1:50547, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T14:16:30,695 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3, hbase.cluster.distributed=false 2024-11-12T14:16:30,731 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:45591-0x1012f67a1e60000, quorum=127.0.0.1:50547, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T14:16:30,738 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45591 2024-11-12T14:16:30,738 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45591 2024-11-12T14:16:30,739 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45591 2024-11-12T14:16:30,740 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45591 2024-11-12T14:16:30,740 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45591 2024-11-12T14:16:30,860 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/2b6d221c5cde:0 server-side Connection retries=45 2024-11-12T14:16:30,862 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:16:30,862 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-12T14:16:30,862 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-12T14:16:30,862 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-12T14:16:30,862 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-12T14:16:30,865 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-12T14:16:30,868 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-12T14:16:30,869 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:42925 2024-11-12T14:16:30,872 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42925 connecting to ZooKeeper ensemble=127.0.0.1:50547 2024-11-12T14:16:30,874 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T14:16:30,879 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T14:16:30,913 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:429250x0, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-12T14:16:30,914 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42925-0x1012f67a1e60001 connected 2024-11-12T14:16:30,915 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42925-0x1012f67a1e60001, quorum=127.0.0.1:50547, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T14:16:30,921 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-12T14:16:30,937 DEBUG [Time-limited test {}] mob.MobFileCache(123): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-12T14:16:30,941 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42925-0x1012f67a1e60001, quorum=127.0.0.1:50547, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-12T14:16:30,948 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42925-0x1012f67a1e60001, quorum=127.0.0.1:50547, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-12T14:16:30,956 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42925 2024-11-12T14:16:30,958 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42925 2024-11-12T14:16:30,959 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42925 2024-11-12T14:16:30,960 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42925 2024-11-12T14:16:30,963 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42925 2024-11-12T14:16:30,986 DEBUG [M:0;2b6d221c5cde:45591 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;2b6d221c5cde:45591 2024-11-12T14:16:30,988 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/2b6d221c5cde,45591,1731420989939 2024-11-12T14:16:31,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45591-0x1012f67a1e60000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T14:16:31,006 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42925-0x1012f67a1e60001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T14:16:31,010 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45591-0x1012f67a1e60000, quorum=127.0.0.1:50547, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/2b6d221c5cde,45591,1731420989939 2024-11-12T14:16:31,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42925-0x1012f67a1e60001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-12T14:16:31,050 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45591-0x1012f67a1e60000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:16:31,051 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42925-0x1012f67a1e60001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:16:31,052 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:45591-0x1012f67a1e60000, quorum=127.0.0.1:50547, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-12T14:16:31,059 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/2b6d221c5cde,45591,1731420989939 from backup master directory 2024-11-12T14:16:31,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45591-0x1012f67a1e60000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/2b6d221c5cde,45591,1731420989939 2024-11-12T14:16:31,139 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45591-0x1012f67a1e60000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T14:16:31,140 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42925-0x1012f67a1e60001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-12T14:16:31,141 WARN [master/2b6d221c5cde:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T14:16:31,141 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=2b6d221c5cde,45591,1731420989939 2024-11-12T14:16:31,145 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-12T14:16:31,147 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-12T14:16:31,223 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/hbase.id] with ID: c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a 2024-11-12T14:16:31,223 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/.tmp/hbase.id 2024-11-12T14:16:31,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741826_1002 (size=42) 2024-11-12T14:16:31,641 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/.tmp/hbase.id]:[hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/hbase.id] 2024-11-12T14:16:31,717 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-12T14:16:31,722 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-12T14:16:31,740 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-12T14:16:31,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42925-0x1012f67a1e60001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:16:31,777 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45591-0x1012f67a1e60000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:16:31,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741827_1003 (size=196) 2024-11-12T14:16:32,235 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T14:16:32,246 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-12T14:16:32,269 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:32,275 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:16:32,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741828_1004 (size=1189) 2024-11-12T14:16:32,760 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/MasterData/data/master/store 2024-11-12T14:16:32,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741829_1005 (size=34) 2024-11-12T14:16:33,223 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-12T14:16:33,226 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:16:33,228 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T14:16:33,228 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:16:33,229 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:16:33,231 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T14:16:33,231 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:16:33,231 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:16:33,233 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731420993228Disabling compacts and flushes for region at 1731420993228Disabling writes for close at 1731420993231 (+3 ms)Writing region close event to WAL at 1731420993231Closed at 1731420993231 2024-11-12T14:16:33,235 WARN [master/2b6d221c5cde:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/MasterData/data/master/store/.initializing 2024-11-12T14:16:33,236 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/MasterData/WALs/2b6d221c5cde,45591,1731420989939 2024-11-12T14:16:33,246 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T14:16:33,269 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2b6d221c5cde%2C45591%2C1731420989939, suffix=, logDir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/MasterData/WALs/2b6d221c5cde,45591,1731420989939, archiveDir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/MasterData/oldWALs, maxLogs=10 2024-11-12T14:16:33,301 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/MasterData/WALs/2b6d221c5cde,45591,1731420989939/2b6d221c5cde%2C45591%2C1731420989939.1731420993275, exclude list is [], retry=0 2024-11-12T14:16:33,331 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42533,DS-7eacfa33-eb6e-4d58-855b-667b624c8b17,DISK] 2024-11-12T14:16:33,335 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-12T14:16:33,379 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/MasterData/WALs/2b6d221c5cde,45591,1731420989939/2b6d221c5cde%2C45591%2C1731420989939.1731420993275 2024-11-12T14:16:33,380 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40953:40953)] 2024-11-12T14:16:33,380 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:16:33,381 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:16:33,385 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:16:33,386 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:16:33,428 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:16:33,464 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-12T14:16:33,472 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:16:33,480 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:16:33,484 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:16:33,490 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-12T14:16:33,490 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:16:33,496 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:16:33,497 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:16:33,502 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-12T14:16:33,503 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:16:33,504 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:16:33,505 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:16:33,511 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-12T14:16:33,512 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:16:33,513 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:16:33,514 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:16:33,520 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:16:33,522 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:16:33,530 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:16:33,531 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:16:33,540 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-12T14:16:33,547 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-12T14:16:33,566 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:16:33,568 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73154052, jitterRate=0.09008032083511353}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-12T14:16:33,585 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731420993399Initializing all the Stores at 1731420993402 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731420993403 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731420993403Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731420993404 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731420993404Cleaning up temporary data from old regions at 1731420993531 (+127 ms)Region opened successfully at 1731420993585 (+54 ms) 2024-11-12T14:16:33,589 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-12T14:16:33,660 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32964c00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2b6d221c5cde/172.17.0.3:0 2024-11-12T14:16:33,709 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-12T14:16:33,723 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-12T14:16:33,723 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-12T14:16:33,726 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-12T14:16:33,728 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-12T14:16:33,734 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-12T14:16:33,734 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-12T14:16:33,768 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-12T14:16:33,781 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45591-0x1012f67a1e60000, quorum=127.0.0.1:50547, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-12T14:16:33,831 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-12T14:16:33,834 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-12T14:16:33,837 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45591-0x1012f67a1e60000, quorum=127.0.0.1:50547, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-12T14:16:33,850 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-12T14:16:33,853 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-12T14:16:33,858 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45591-0x1012f67a1e60000, quorum=127.0.0.1:50547, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-12T14:16:33,873 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-12T14:16:33,878 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45591-0x1012f67a1e60000, quorum=127.0.0.1:50547, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-12T14:16:33,890 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-12T14:16:33,916 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:45591-0x1012f67a1e60000, quorum=127.0.0.1:50547, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-12T14:16:33,928 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-12T14:16:33,994 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42925-0x1012f67a1e60001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T14:16:33,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45591-0x1012f67a1e60000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-12T14:16:33,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42925-0x1012f67a1e60001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:16:33,995 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45591-0x1012f67a1e60000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:16:33,998 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=2b6d221c5cde,45591,1731420989939, sessionid=0x1012f67a1e60000, setting cluster-up flag (Was=false) 2024-11-12T14:16:34,134 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42925-0x1012f67a1e60001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:16:34,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45591-0x1012f67a1e60000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:16:34,260 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-12T14:16:34,274 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2b6d221c5cde,45591,1731420989939 2024-11-12T14:16:34,302 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45591-0x1012f67a1e60000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:16:34,304 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42925-0x1012f67a1e60001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:16:34,344 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-12T14:16:34,357 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=2b6d221c5cde,45591,1731420989939 2024-11-12T14:16:34,371 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-12T14:16:34,404 INFO [RS:0;2b6d221c5cde:42925 {}] regionserver.HRegionServer(746): ClusterId : c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a 2024-11-12T14:16:34,413 DEBUG [RS:0;2b6d221c5cde:42925 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-12T14:16:34,435 DEBUG [RS:0;2b6d221c5cde:42925 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-12T14:16:34,435 DEBUG [RS:0;2b6d221c5cde:42925 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-12T14:16:34,448 DEBUG [RS:0;2b6d221c5cde:42925 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-12T14:16:34,449 DEBUG [RS:0;2b6d221c5cde:42925 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46b6205c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=2b6d221c5cde/172.17.0.3:0 2024-11-12T14:16:34,478 DEBUG [RS:0;2b6d221c5cde:42925 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;2b6d221c5cde:42925 2024-11-12T14:16:34,483 INFO [RS:0;2b6d221c5cde:42925 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-12T14:16:34,483 INFO [RS:0;2b6d221c5cde:42925 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-12T14:16:34,483 DEBUG [RS:0;2b6d221c5cde:42925 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-12T14:16:34,485 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-12T14:16:34,494 INFO [RS:0;2b6d221c5cde:42925 {}] regionserver.HRegionServer(2659): reportForDuty to master=2b6d221c5cde,45591,1731420989939 with port=42925, startcode=1731420990822 2024-11-12T14:16:34,499 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-12T14:16:34,510 DEBUG [RS:0;2b6d221c5cde:42925 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-12T14:16:34,521 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-12T14:16:34,529 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 2b6d221c5cde,45591,1731420989939 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-12T14:16:34,540 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/2b6d221c5cde:0, corePoolSize=5, maxPoolSize=5 2024-11-12T14:16:34,540 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/2b6d221c5cde:0, corePoolSize=5, maxPoolSize=5 2024-11-12T14:16:34,564 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/2b6d221c5cde:0, corePoolSize=5, maxPoolSize=5 2024-11-12T14:16:34,565 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/2b6d221c5cde:0, corePoolSize=5, maxPoolSize=5 2024-11-12T14:16:34,566 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/2b6d221c5cde:0, corePoolSize=10, maxPoolSize=10 2024-11-12T14:16:34,566 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/2b6d221c5cde:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:16:34,566 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/2b6d221c5cde:0, corePoolSize=2, maxPoolSize=2 2024-11-12T14:16:34,566 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/2b6d221c5cde:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:16:34,580 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:39419, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-12T14:16:34,587 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45591 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:34,598 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T14:16:34,598 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-12T14:16:34,611 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:16:34,611 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-12T14:16:34,617 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731421024617 2024-11-12T14:16:34,619 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-12T14:16:34,620 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-12T14:16:34,624 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-12T14:16:34,625 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-12T14:16:34,625 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-12T14:16:34,625 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-12T14:16:34,631 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T14:16:34,637 DEBUG [RS:0;2b6d221c5cde:42925 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-12T14:16:34,637 WARN [RS:0;2b6d221c5cde:42925 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-12T14:16:34,643 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-12T14:16:34,645 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-12T14:16:34,645 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-12T14:16:34,659 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-12T14:16:34,660 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-12T14:16:34,668 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/2b6d221c5cde:0:becomeActiveMaster-HFileCleaner.large.0-1731420994662,5,FailOnTimeoutGroup] 2024-11-12T14:16:34,670 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/2b6d221c5cde:0:becomeActiveMaster-HFileCleaner.small.0-1731420994669,5,FailOnTimeoutGroup] 2024-11-12T14:16:34,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741831_1007 (size=1321) 2024-11-12T14:16:34,670 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-12T14:16:34,670 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-12T14:16:34,678 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-12T14:16:34,678 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-12T14:16:34,738 INFO [RS:0;2b6d221c5cde:42925 {}] regionserver.HRegionServer(2659): reportForDuty to master=2b6d221c5cde,45591,1731420989939 with port=42925, startcode=1731420990822 2024-11-12T14:16:34,741 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45591 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:34,743 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45591 {}] master.ServerManager(517): Registering regionserver=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:34,752 DEBUG [RS:0;2b6d221c5cde:42925 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3 2024-11-12T14:16:34,752 DEBUG [RS:0;2b6d221c5cde:42925 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34349 2024-11-12T14:16:34,752 DEBUG [RS:0;2b6d221c5cde:42925 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-12T14:16:34,766 DEBUG [RS:0;2b6d221c5cde:42925 {}] zookeeper.ZKUtil(111): regionserver:42925-0x1012f67a1e60001, quorum=127.0.0.1:50547, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:34,766 WARN [RS:0;2b6d221c5cde:42925 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-12T14:16:34,766 INFO [RS:0;2b6d221c5cde:42925 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:16:34,767 DEBUG [RS:0;2b6d221c5cde:42925 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/WALs/2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:34,768 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45591-0x1012f67a1e60000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T14:16:34,772 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [2b6d221c5cde,42925,1731420990822] 2024-11-12T14:16:34,798 INFO [RS:0;2b6d221c5cde:42925 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-12T14:16:34,825 INFO [RS:0;2b6d221c5cde:42925 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-12T14:16:34,833 INFO [RS:0;2b6d221c5cde:42925 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-12T14:16:34,833 INFO [RS:0;2b6d221c5cde:42925 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:16:34,835 INFO [RS:0;2b6d221c5cde:42925 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-12T14:16:34,853 INFO [RS:0;2b6d221c5cde:42925 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-12T14:16:34,855 INFO [RS:0;2b6d221c5cde:42925 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-12T14:16:34,856 DEBUG [RS:0;2b6d221c5cde:42925 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/2b6d221c5cde:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:16:34,856 DEBUG [RS:0;2b6d221c5cde:42925 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/2b6d221c5cde:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:16:34,856 DEBUG [RS:0;2b6d221c5cde:42925 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/2b6d221c5cde:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:16:34,856 DEBUG [RS:0;2b6d221c5cde:42925 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/2b6d221c5cde:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:16:34,857 DEBUG [RS:0;2b6d221c5cde:42925 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/2b6d221c5cde:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:16:34,857 DEBUG [RS:0;2b6d221c5cde:42925 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/2b6d221c5cde:0, corePoolSize=2, maxPoolSize=2 2024-11-12T14:16:34,857 DEBUG [RS:0;2b6d221c5cde:42925 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:16:34,857 DEBUG [RS:0;2b6d221c5cde:42925 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/2b6d221c5cde:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:16:34,857 DEBUG [RS:0;2b6d221c5cde:42925 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/2b6d221c5cde:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:16:34,858 DEBUG [RS:0;2b6d221c5cde:42925 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/2b6d221c5cde:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:16:34,858 DEBUG [RS:0;2b6d221c5cde:42925 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/2b6d221c5cde:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:16:34,858 DEBUG [RS:0;2b6d221c5cde:42925 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/2b6d221c5cde:0, corePoolSize=1, maxPoolSize=1 2024-11-12T14:16:34,858 DEBUG [RS:0;2b6d221c5cde:42925 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/2b6d221c5cde:0, corePoolSize=3, maxPoolSize=3 2024-11-12T14:16:34,858 DEBUG [RS:0;2b6d221c5cde:42925 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0, corePoolSize=3, maxPoolSize=3 2024-11-12T14:16:34,876 INFO [RS:0;2b6d221c5cde:42925 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T14:16:34,877 INFO [RS:0;2b6d221c5cde:42925 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-12T14:16:34,877 INFO [RS:0;2b6d221c5cde:42925 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:16:34,877 INFO [RS:0;2b6d221c5cde:42925 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-12T14:16:34,877 INFO [RS:0;2b6d221c5cde:42925 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-12T14:16:34,877 INFO [RS:0;2b6d221c5cde:42925 {}] hbase.ChoreService(168): Chore ScheduledChore name=2b6d221c5cde,42925,1731420990822-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T14:16:34,916 INFO [RS:0;2b6d221c5cde:42925 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-12T14:16:34,919 INFO [RS:0;2b6d221c5cde:42925 {}] hbase.ChoreService(168): Chore ScheduledChore name=2b6d221c5cde,42925,1731420990822-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:16:34,919 INFO [RS:0;2b6d221c5cde:42925 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:16:34,919 INFO [RS:0;2b6d221c5cde:42925 {}] regionserver.Replication(171): 2b6d221c5cde,42925,1731420990822 started 2024-11-12T14:16:34,948 INFO [RS:0;2b6d221c5cde:42925 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:16:34,948 INFO [RS:0;2b6d221c5cde:42925 {}] regionserver.HRegionServer(1482): Serving as 2b6d221c5cde,42925,1731420990822, RpcServer on 2b6d221c5cde/172.17.0.3:42925, sessionid=0x1012f67a1e60001 2024-11-12T14:16:34,949 DEBUG [RS:0;2b6d221c5cde:42925 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-12T14:16:34,950 DEBUG [RS:0;2b6d221c5cde:42925 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:34,950 DEBUG [RS:0;2b6d221c5cde:42925 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2b6d221c5cde,42925,1731420990822' 2024-11-12T14:16:34,950 DEBUG [RS:0;2b6d221c5cde:42925 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-12T14:16:34,952 DEBUG [RS:0;2b6d221c5cde:42925 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-12T14:16:34,953 DEBUG [RS:0;2b6d221c5cde:42925 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-12T14:16:34,953 DEBUG [RS:0;2b6d221c5cde:42925 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-12T14:16:34,954 DEBUG [RS:0;2b6d221c5cde:42925 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:34,954 DEBUG [RS:0;2b6d221c5cde:42925 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '2b6d221c5cde,42925,1731420990822' 2024-11-12T14:16:34,954 DEBUG [RS:0;2b6d221c5cde:42925 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-12T14:16:34,954 DEBUG [RS:0;2b6d221c5cde:42925 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-12T14:16:34,957 DEBUG [RS:0;2b6d221c5cde:42925 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-12T14:16:34,957 INFO [RS:0;2b6d221c5cde:42925 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-12T14:16:34,957 INFO [RS:0;2b6d221c5cde:42925 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-12T14:16:35,063 INFO [RS:0;2b6d221c5cde:42925 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-12T14:16:35,067 INFO [RS:0;2b6d221c5cde:42925 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2b6d221c5cde%2C42925%2C1731420990822, suffix=, logDir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/WALs/2b6d221c5cde,42925,1731420990822, archiveDir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/oldWALs, maxLogs=32 2024-11-12T14:16:35,070 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-12T14:16:35,070 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3 2024-11-12T14:16:35,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741832_1008 (size=32) 2024-11-12T14:16:35,088 DEBUG [RS:0;2b6d221c5cde:42925 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/WALs/2b6d221c5cde,42925,1731420990822/2b6d221c5cde%2C42925%2C1731420990822.1731420995070, exclude list is [], retry=0 2024-11-12T14:16:35,090 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:16:35,093 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T14:16:35,097 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T14:16:35,097 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:16:35,099 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:16:35,100 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42533,DS-7eacfa33-eb6e-4d58-855b-667b624c8b17,DISK] 2024-11-12T14:16:35,102 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T14:16:35,107 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T14:16:35,107 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:16:35,108 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:16:35,108 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T14:16:35,108 INFO [RS:0;2b6d221c5cde:42925 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/WALs/2b6d221c5cde,42925,1731420990822/2b6d221c5cde%2C42925%2C1731420990822.1731420995070 2024-11-12T14:16:35,111 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T14:16:35,111 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:16:35,113 DEBUG [RS:0;2b6d221c5cde:42925 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40953:40953)] 2024-11-12T14:16:35,113 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:16:35,113 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T14:16:35,116 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T14:16:35,116 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:16:35,118 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:16:35,118 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T14:16:35,119 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/hbase/meta/1588230740 2024-11-12T14:16:35,120 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/hbase/meta/1588230740 2024-11-12T14:16:35,124 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T14:16:35,124 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T14:16:35,126 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T14:16:35,129 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T14:16:35,136 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:16:35,141 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68641055, jitterRate=0.022831425070762634}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T14:16:35,146 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731420995090Initializing all the Stores at 1731420995093 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731420995093Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731420995093Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731420995093Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731420995093Cleaning up temporary data from old regions at 1731420995124 (+31 ms)Region opened successfully at 1731420995145 (+21 ms) 2024-11-12T14:16:35,146 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T14:16:35,146 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T14:16:35,146 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T14:16:35,147 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T14:16:35,147 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T14:16:35,150 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T14:16:35,150 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731420995146Disabling compacts and flushes for region at 1731420995146Disabling writes for close at 1731420995147 (+1 ms)Writing region close event to WAL at 1731420995149 (+2 ms)Closed at 1731420995150 (+1 ms) 2024-11-12T14:16:35,155 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T14:16:35,155 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-12T14:16:35,163 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-12T14:16:35,174 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T14:16:35,177 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-12T14:16:35,331 DEBUG [2b6d221c5cde:45591 {}] assignment.AssignmentManager(2464): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-12T14:16:35,344 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:35,352 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2b6d221c5cde,42925,1731420990822, state=OPENING 2024-11-12T14:16:35,376 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-12T14:16:35,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42925-0x1012f67a1e60001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:16:35,386 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45591-0x1012f67a1e60000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:16:35,387 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:16:35,387 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:16:35,389 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-12T14:16:35,391 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=2b6d221c5cde,42925,1731420990822}] 2024-11-12T14:16:35,569 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-12T14:16:35,573 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35007, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-12T14:16:35,587 INFO [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-12T14:16:35,587 INFO [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-12T14:16:35,588 INFO [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-12T14:16:35,593 INFO [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=2b6d221c5cde%2C42925%2C1731420990822.meta, suffix=.meta, logDir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/WALs/2b6d221c5cde,42925,1731420990822, archiveDir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/oldWALs, maxLogs=32 2024-11-12T14:16:35,616 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/WALs/2b6d221c5cde,42925,1731420990822/2b6d221c5cde%2C42925%2C1731420990822.meta.1731420995596.meta, exclude list is [], retry=0 2024-11-12T14:16:35,620 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:42533,DS-7eacfa33-eb6e-4d58-855b-667b624c8b17,DISK] 2024-11-12T14:16:35,624 INFO [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/WALs/2b6d221c5cde,42925,1731420990822/2b6d221c5cde%2C42925%2C1731420990822.meta.1731420995596.meta 2024-11-12T14:16:35,624 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:40953:40953)] 2024-11-12T14:16:35,625 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:16:35,627 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-12T14:16:35,630 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-12T14:16:35,636 INFO [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-12T14:16:35,641 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-12T14:16:35,642 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:16:35,642 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-12T14:16:35,643 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-12T14:16:35,646 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-12T14:16:35,648 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-12T14:16:35,649 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:16:35,650 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:16:35,650 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-12T14:16:35,654 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-12T14:16:35,654 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:16:35,655 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:16:35,655 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-12T14:16:35,658 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-12T14:16:35,658 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:16:35,661 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:16:35,661 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-12T14:16:35,663 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-12T14:16:35,663 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:16:35,664 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-12T14:16:35,667 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-12T14:16:35,669 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/hbase/meta/1588230740 2024-11-12T14:16:35,672 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/hbase/meta/1588230740 2024-11-12T14:16:35,675 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-12T14:16:35,675 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-12T14:16:35,676 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T14:16:35,680 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-12T14:16:35,683 INFO [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64090072, jitterRate=-0.044983506202697754}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T14:16:35,683 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-12T14:16:35,685 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731420995643Writing region info on filesystem at 1731420995643Initializing all the Stores at 1731420995646 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731420995646Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731420995646Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731420995646Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731420995646Cleaning up temporary data from old regions at 1731420995675 (+29 ms)Running coprocessor post-open hooks at 1731420995683 (+8 ms)Region opened successfully at 1731420995685 (+2 ms) 2024-11-12T14:16:35,695 INFO [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731420995559 2024-11-12T14:16:35,708 DEBUG [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-12T14:16:35,708 INFO [RS_OPEN_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-12T14:16:35,710 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:35,714 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 2b6d221c5cde,42925,1731420990822, state=OPEN 2024-11-12T14:16:35,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45591-0x1012f67a1e60000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:16:35,798 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42925-0x1012f67a1e60001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-12T14:16:35,799 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:16:35,799 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-12T14:16:35,799 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:35,805 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-12T14:16:35,806 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=2b6d221c5cde,42925,1731420990822 in 408 msec 2024-11-12T14:16:35,815 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-12T14:16:35,816 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 645 msec 2024-11-12T14:16:35,818 DEBUG [PEWorker-2 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-12T14:16:35,818 INFO [PEWorker-2 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-12T14:16:35,846 DEBUG [PEWorker-2 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:16:35,847 DEBUG [PEWorker-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:16:35,872 DEBUG [PEWorker-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:16:35,875 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:50139, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:16:35,910 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.4910 sec 2024-11-12T14:16:35,910 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731420995910, completionTime=-1 2024-11-12T14:16:35,913 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-12T14:16:35,914 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] assignment.AssignmentManager(1756): Joining cluster... 2024-11-12T14:16:35,955 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] assignment.AssignmentManager(1768): Number of RegionServers=1 2024-11-12T14:16:35,955 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731421055955 2024-11-12T14:16:35,955 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731421115955 2024-11-12T14:16:35,955 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] assignment.AssignmentManager(1775): Joined the cluster in 41 msec 2024-11-12T14:16:35,958 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2b6d221c5cde,45591,1731420989939-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-12T14:16:35,959 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2b6d221c5cde,45591,1731420989939-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:16:35,959 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2b6d221c5cde,45591,1731420989939-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:16:35,961 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-2b6d221c5cde:45591, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:16:35,962 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-12T14:16:35,963 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-12T14:16:35,973 DEBUG [master/2b6d221c5cde:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-12T14:16:36,004 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.863sec 2024-11-12T14:16:36,006 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-12T14:16:36,010 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-12T14:16:36,011 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-12T14:16:36,011 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-12T14:16:36,011 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-12T14:16:36,013 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2b6d221c5cde,45591,1731420989939-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-12T14:16:36,016 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2b6d221c5cde,45591,1731420989939-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-12T14:16:36,040 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-12T14:16:36,041 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-12T14:16:36,042 INFO [master/2b6d221c5cde:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=2b6d221c5cde,45591,1731420989939-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-12T14:16:36,089 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@764be56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:36,092 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-12T14:16:36,092 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-12T14:16:36,098 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:16:36,101 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:16:36,133 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:16:36,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:16:36,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:16:36,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79bfeeb1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:36,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:16:36,141 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:16:36,146 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:16:36,151 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35292, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:16:36,161 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1439c7ad, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:36,162 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:16:36,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:16:36,177 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:16:36,186 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:16:36,224 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=2b6d221c5cde,45591,1731420989939 2024-11-12T14:16:36,242 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=813, ProcessCount=11, AvailableMemoryMB=2651 2024-11-12T14:16:36,258 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T14:16:36,364 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 2b6d221c5cde,45591,1731420989939 2024-11-12T14:16:36,367 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@249d22a 2024-11-12T14:16:36,373 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T14:16:36,412 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35296, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T14:16:36,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T14:16:36,424 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T14:16:36,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T14:16:36,448 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T14:16:36,451 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-12T14:16:36,451 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:16:36,457 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T14:16:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T14:16:36,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741835_1011 (size=1061) 2024-11-12T14:16:36,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T14:16:36,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T14:16:36,934 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3 2024-11-12T14:16:36,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741836_1012 (size=53) 2024-11-12T14:16:36,975 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:16:36,976 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6ea61e7c1d124f78e4a5241afdaab254, disabling compactions & flushes 2024-11-12T14:16:36,976 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:36,976 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:36,976 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. after waiting 0 ms 2024-11-12T14:16:36,976 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:36,976 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:36,976 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6ea61e7c1d124f78e4a5241afdaab254: Waiting for close lock at 1731420996976Disabling compacts and flushes for region at 1731420996976Disabling writes for close at 1731420996976Writing region close event to WAL at 1731420996976Closed at 1731420996976 2024-11-12T14:16:36,987 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T14:16:36,994 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731420996987"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731420996987"}]},"ts":"1731420996987"} 2024-11-12T14:16:37,007 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T14:16:37,012 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T14:16:37,016 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731420997013"}]},"ts":"1731420997013"} 2024-11-12T14:16:37,029 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T14:16:37,035 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ea61e7c1d124f78e4a5241afdaab254, ASSIGN}] 2024-11-12T14:16:37,039 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ea61e7c1d124f78e4a5241afdaab254, ASSIGN 2024-11-12T14:16:37,045 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ea61e7c1d124f78e4a5241afdaab254, ASSIGN; state=OFFLINE, location=2b6d221c5cde,42925,1731420990822; forceNewPlan=false, retain=false 2024-11-12T14:16:37,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T14:16:37,201 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6ea61e7c1d124f78e4a5241afdaab254, regionState=OPENING, regionLocation=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:37,213 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ea61e7c1d124f78e4a5241afdaab254, ASSIGN because future has completed 2024-11-12T14:16:37,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822}] 2024-11-12T14:16:37,399 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:37,400 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:16:37,400 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:37,400 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:16:37,401 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:37,401 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:37,416 INFO [StoreOpener-6ea61e7c1d124f78e4a5241afdaab254-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:37,442 INFO [StoreOpener-6ea61e7c1d124f78e4a5241afdaab254-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T14:16:37,443 INFO [StoreOpener-6ea61e7c1d124f78e4a5241afdaab254-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ea61e7c1d124f78e4a5241afdaab254 columnFamilyName A 2024-11-12T14:16:37,443 DEBUG [StoreOpener-6ea61e7c1d124f78e4a5241afdaab254-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:16:37,452 INFO [StoreOpener-6ea61e7c1d124f78e4a5241afdaab254-1 {}] regionserver.HStore(327): Store=6ea61e7c1d124f78e4a5241afdaab254/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:16:37,452 INFO [StoreOpener-6ea61e7c1d124f78e4a5241afdaab254-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:37,455 INFO [StoreOpener-6ea61e7c1d124f78e4a5241afdaab254-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T14:16:37,456 INFO [StoreOpener-6ea61e7c1d124f78e4a5241afdaab254-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ea61e7c1d124f78e4a5241afdaab254 columnFamilyName B 2024-11-12T14:16:37,456 DEBUG [StoreOpener-6ea61e7c1d124f78e4a5241afdaab254-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:16:37,457 INFO [StoreOpener-6ea61e7c1d124f78e4a5241afdaab254-1 {}] regionserver.HStore(327): Store=6ea61e7c1d124f78e4a5241afdaab254/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:16:37,458 INFO [StoreOpener-6ea61e7c1d124f78e4a5241afdaab254-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:37,464 INFO [StoreOpener-6ea61e7c1d124f78e4a5241afdaab254-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T14:16:37,465 INFO [StoreOpener-6ea61e7c1d124f78e4a5241afdaab254-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6ea61e7c1d124f78e4a5241afdaab254 columnFamilyName C 2024-11-12T14:16:37,466 DEBUG [StoreOpener-6ea61e7c1d124f78e4a5241afdaab254-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:16:37,469 INFO [StoreOpener-6ea61e7c1d124f78e4a5241afdaab254-1 {}] regionserver.HStore(327): Store=6ea61e7c1d124f78e4a5241afdaab254/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:16:37,470 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:37,470 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:37,472 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:37,473 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:37,475 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:37,476 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:37,477 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T14:16:37,480 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:37,500 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:16:37,505 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 6ea61e7c1d124f78e4a5241afdaab254; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61264159, jitterRate=-0.08709289133548737}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T14:16:37,505 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:37,508 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 6ea61e7c1d124f78e4a5241afdaab254: Running coprocessor pre-open hook at 1731420997401Writing region info on filesystem at 1731420997401Initializing all the Stores at 1731420997409 (+8 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731420997409Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731420997411 (+2 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731420997411Cleaning up temporary data from old regions at 1731420997476 (+65 ms)Running coprocessor post-open hooks at 1731420997505 (+29 ms)Region opened successfully at 1731420997506 (+1 ms) 2024-11-12T14:16:37,513 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., pid=6, masterSystemTime=1731420997384 2024-11-12T14:16:37,524 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:37,524 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:37,525 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=6ea61e7c1d124f78e4a5241afdaab254, regionState=OPEN, openSeqNum=2, regionLocation=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:37,532 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 because future has completed 2024-11-12T14:16:37,541 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45591 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=2b6d221c5cde,42925,1731420990822, table=TestAcidGuarantees, region=6ea61e7c1d124f78e4a5241afdaab254. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T14:16:37,555 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-12T14:16:37,566 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 in 331 msec 2024-11-12T14:16:37,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-12T14:16:37,577 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ea61e7c1d124f78e4a5241afdaab254, ASSIGN in 520 msec 2024-11-12T14:16:37,582 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T14:16:37,582 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731420997582"}]},"ts":"1731420997582"} 2024-11-12T14:16:37,588 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T14:16:37,591 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T14:16:37,603 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1630 sec 2024-11-12T14:16:37,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-12T14:16:37,633 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:16:37,640 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ab38911, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:37,640 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:16:37,640 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:16:37,644 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:16:37,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:16:37,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:16:37,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63473470, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:37,645 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:16:37,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:16:37,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:16:37,652 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35314, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:16:37,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@302d7c14, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:37,666 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:16:37,670 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:16:37,671 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:16:37,674 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:16:37,677 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T14:16:37,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 2b6d221c5cde,45591,1731420989939 2024-11-12T14:16:37,681 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@d32c91 2024-11-12T14:16:37,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T14:16:37,690 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35318, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T14:16:37,700 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dbe51c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:37,701 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:16:37,701 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:16:37,703 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:16:37,704 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:16:37,704 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:16:37,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35b83ee0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:37,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:16:37,705 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:16:37,706 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:16:37,709 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35340, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:16:37,711 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@797de42a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:37,717 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ec95bee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:37,717 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:16:37,717 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:16:37,726 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:16:37,732 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:16:37,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:16:37,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70a888f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:37,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:16:37,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:16:37,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:16:37,744 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35360, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:16:37,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6da9de27, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:37,758 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@639d2f4e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:37,758 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:16:37,758 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:16:37,768 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:16:37,777 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:16:37,778 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:16:37,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b2e42a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:37,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:16:37,779 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:16:37,780 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:16:37,785 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35368, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:16:37,794 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4aa5770f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:37,795 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@241fa23d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:37,796 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:16:37,796 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:16:37,806 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:16:37,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:16:37,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:16:37,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b232dd7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:37,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:16:37,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:16:37,813 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:16:37,818 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35390, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:16:37,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30e61703, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:37,829 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6c5ab0ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:37,829 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:16:37,829 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:16:37,833 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:16:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:16:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:16:37,834 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6995889c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:37,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:16:37,835 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:16:37,842 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:16:37,845 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35410, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:16:37,850 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554bc07b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:37,872 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b9cdeaf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:37,873 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:16:37,873 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:16:37,880 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:16:37,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:16:37,882 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:16:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f67bf3a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:16:37,883 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:16:37,884 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:16:37,889 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:16:37,890 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@111e962d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:37,892 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d577fca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:37,892 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:16:37,892 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:16:37,896 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:16:37,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:16:37,896 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:16:37,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f6fe38c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:37,898 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:16:37,899 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:16:37,900 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:16:37,905 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:16:37,907 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60da3efe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:37,909 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4bda0ae1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:37,910 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:16:37,910 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:16:37,913 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:16:37,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:16:37,914 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:16:37,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d9487f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:37,921 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:16:37,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:16:37,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:16:37,931 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35482, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:16:37,937 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff7d382, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:37,945 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59025020, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:37,945 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:16:37,945 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:16:37,949 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:16:37,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:16:37,950 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:16:37,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ee249d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:37,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:16:37,951 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:16:37,952 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:16:37,961 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35494, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:16:37,966 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c9e0f5f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:16:37,979 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:16:37,980 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:16:37,981 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:16:37,982 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:16:37,982 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:16:37,985 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57718, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:16:37,986 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:16:37,997 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:16:38,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:16:38,000 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:16:38,000 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:16:38,001 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:16:38,001 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:16:38,002 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:16:38,003 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:16:38,003 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57736, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:16:38,004 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:16:38,004 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2] 2024-11-12T14:16:38,007 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:16:38,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:16:38,010 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:16:38,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:16:38,010 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57756, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:16:38,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2] 2024-11-12T14:16:38,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2] 2024-11-12T14:16:38,015 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57758, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:16:38,016 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2] 2024-11-12T14:16:38,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:16:38,021 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:16:38,023 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:16:38,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-12T14:16:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:16:38,030 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:16:38,030 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:16:38,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2] 2024-11-12T14:16:38,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T14:16:38,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2] 2024-11-12T14:16:38,035 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:16:38,035 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:16:38,036 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:16:38,036 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:16:38,038 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:16:38,040 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:16:38,041 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:16:38,046 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57786, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:16:38,048 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2] 2024-11-12T14:16:38,052 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2] 2024-11-12T14:16:38,052 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57778, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:16:38,060 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:16:38,069 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:16:38,075 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2] 2024-11-12T14:16:38,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T14:16:38,185 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ea61e7c1d124f78e4a5241afdaab254 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T14:16:38,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:38,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=A 2024-11-12T14:16:38,210 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:38,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=B 2024-11-12T14:16:38,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:38,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=C 2024-11-12T14:16:38,211 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:38,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T14:16:38,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:38,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:38,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:38,274 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:38,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:38,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:38,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T14:16:38,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/08feba09c95c43b484e9b2d0ab6fb503 is 50, key is test_row_0/A:col10/1731420998155/Put/seqid=0 2024-11-12T14:16:38,462 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T14:16:38,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:38,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:38,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:38,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:38,486 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:38,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:38,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741837_1013 (size=14341) 2024-11-12T14:16:38,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/08feba09c95c43b484e9b2d0ab6fb503 2024-11-12T14:16:38,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T14:16:38,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T14:16:38,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:38,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:38,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:38,665 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:38,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:38,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:38,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/44505f337484442da4f854bbab0a652b is 50, key is test_row_0/B:col10/1731420998155/Put/seqid=0 2024-11-12T14:16:38,841 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T14:16:38,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:38,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:38,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:38,843 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:38,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:38,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:38,891 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731420998533, max timestamp=1731420998684Num uniques 859; 2024-11-12T14:16:38,893 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T14:16:38,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731420998533, max timestamp=1731420998646Num uniques 859; 2024-11-12T14:16:38,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T14:16:38,924 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731420998533, max timestamp=1731420998633Num uniques 859; 2024-11-12T14:16:38,925 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-12T14:16:38,932 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741838_1014 (size=12001) 2024-11-12T14:16:38,934 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/44505f337484442da4f854bbab0a652b 2024-11-12T14:16:39,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T14:16:39,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:39,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:39,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:39,022 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:39,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:39,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:39,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/ae7f1e5c1b90408cb4aa1f895ae2e9d4 is 50, key is test_row_0/C:col10/1731420998155/Put/seqid=0 2024-11-12T14:16:39,097 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741839_1015 (size=12001) 2024-11-12T14:16:39,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:39,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:39,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421059030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:39,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421059034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:39,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:39,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421059080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:39,110 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,111 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:39,110 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,112 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:39,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:39,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:39,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421059095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:39,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,130 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:39,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:39,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421059097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:39,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,131 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:39,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T14:16:39,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T14:16:39,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:39,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:39,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:39,194 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:39,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:39,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:39,243 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:39,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421059222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:39,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:39,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421059224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:39,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:39,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421059226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:39,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:39,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,250 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:39,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:39,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421059246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:39,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,267 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,268 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:39,280 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:39,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421059242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:39,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,285 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:39,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T14:16:39,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:39,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:39,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:39,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:39,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:39,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:39,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:39,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421059461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:39,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,471 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:39,471 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:39,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421059461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:39,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,474 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:39,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:39,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421059480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:39,491 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:39,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:39,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421059481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:39,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,495 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:39,495 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:39,506 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/ae7f1e5c1b90408cb4aa1f895ae2e9d4 2024-11-12T14:16:39,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421059490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:39,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,508 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:39,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T14:16:39,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:39,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:39,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:39,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:39,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:39,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:39,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/08feba09c95c43b484e9b2d0ab6fb503 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/08feba09c95c43b484e9b2d0ab6fb503 2024-11-12T14:16:39,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/08feba09c95c43b484e9b2d0ab6fb503, entries=200, sequenceid=12, filesize=14.0 K 2024-11-12T14:16:39,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/44505f337484442da4f854bbab0a652b as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/44505f337484442da4f854bbab0a652b 2024-11-12T14:16:39,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/44505f337484442da4f854bbab0a652b, entries=150, sequenceid=12, filesize=11.7 K 2024-11-12T14:16:39,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/ae7f1e5c1b90408cb4aa1f895ae2e9d4 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/ae7f1e5c1b90408cb4aa1f895ae2e9d4 2024-11-12T14:16:39,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/ae7f1e5c1b90408cb4aa1f895ae2e9d4, entries=150, sequenceid=12, filesize=11.7 K 2024-11-12T14:16:39,683 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.08 KB/130125 for 6ea61e7c1d124f78e4a5241afdaab254 in 1497ms, sequenceid=12, compaction requested=false 2024-11-12T14:16:39,685 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-12T14:16:39,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:39,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-12T14:16:39,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:39,705 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 6ea61e7c1d124f78e4a5241afdaab254 3/3 column families, dataSize=127.08 KB heapSize=364.15 KB 2024-11-12T14:16:39,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=A 2024-11-12T14:16:39,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:16:39,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=B 2024-11-12T14:16:39,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:16:39,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=C 2024-11-12T14:16:39,714 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:16:39,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/bae66fce00154e688a47fb8355495de9 is 50, key is test_row_0/A:col10/1731420999005/Put/seqid=0 2024-11-12T14:16:39,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:39,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:39,794 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741840_1016 (size=12001) 2024-11-12T14:16:39,803 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/bae66fce00154e688a47fb8355495de9 2024-11-12T14:16:39,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/b7ddf445cfc04660882023922d42e0d7 is 50, key is test_row_0/B:col10/1731420999005/Put/seqid=0 2024-11-12T14:16:39,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741841_1017 (size=12001) 2024-11-12T14:16:39,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:39,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421059875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:39,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,940 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:39,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:39,948 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,949 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:39,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421059878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:39,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421059878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:39,952 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,953 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:39,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:39,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:39,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421059918, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:39,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:39,987 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:39,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421059936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:39,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:39,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:40,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:40,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421060062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:40,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:40,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:40,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:40,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:40,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421060062, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:40,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:40,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:40,093 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:40,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:40,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421060063, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:40,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:40,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:40,102 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:40,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:40,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421060091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:40,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:40,147 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:40,148 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:40,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421060101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:40,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:40,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:40,150 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:40,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T14:16:40,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:40,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421060302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:40,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:40,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:40,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:40,319 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421060303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:40,320 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:40,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:40,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:40,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:40,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421060313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:40,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:40,326 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:40,327 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:40,337 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-12T14:16:40,338 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-12T14:16:40,341 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/b7ddf445cfc04660882023922d42e0d7 2024-11-12T14:16:40,351 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T14:16:40,351 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-12T14:16:40,351 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-12T14:16:40,352 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-12T14:16:40,361 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-12T14:16:40,361 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-12T14:16:40,375 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:40,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421060363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:40,378 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:40,380 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:40,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421060364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:40,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:40,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:40,383 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:40,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/8bcf3c54b8204ef6a8d9ffe1f140ec5a is 50, key is test_row_0/C:col10/1731420999005/Put/seqid=0 2024-11-12T14:16:40,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741842_1018 (size=12001) 2024-11-12T14:16:40,646 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:40,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421060637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:40,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:40,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:40,649 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:40,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:40,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421060637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:40,654 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:40,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421060640, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:40,656 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:40,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:40,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:40,657 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:40,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:40,657 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:40,706 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:40,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421060693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:40,707 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:40,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:40,708 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:40,711 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:40,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421060700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:40,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:40,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:40,934 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/8bcf3c54b8204ef6a8d9ffe1f140ec5a 2024-11-12T14:16:41,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/bae66fce00154e688a47fb8355495de9 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/bae66fce00154e688a47fb8355495de9 2024-11-12T14:16:41,101 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/bae66fce00154e688a47fb8355495de9, entries=150, sequenceid=48, filesize=11.7 K 2024-11-12T14:16:41,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/b7ddf445cfc04660882023922d42e0d7 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/b7ddf445cfc04660882023922d42e0d7 2024-11-12T14:16:41,137 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/b7ddf445cfc04660882023922d42e0d7, entries=150, sequenceid=48, filesize=11.7 K 2024-11-12T14:16:41,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/8bcf3c54b8204ef6a8d9ffe1f140ec5a as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/8bcf3c54b8204ef6a8d9ffe1f140ec5a 2024-11-12T14:16:41,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:41,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421061161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:41,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:41,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421061161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:41,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:41,180 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:41,181 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:41,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:41,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:41,185 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:41,188 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/8bcf3c54b8204ef6a8d9ffe1f140ec5a, entries=150, sequenceid=48, filesize=11.7 K 2024-11-12T14:16:41,190 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~127.08 KB/130125, heapSize ~364.10 KB/372840, currentSize=60.38 KB/61830 for 6ea61e7c1d124f78e4a5241afdaab254 in 1486ms, sequenceid=48, compaction requested=false 2024-11-12T14:16:41,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:41,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:41,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-12T14:16:41,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-12T14:16:41,207 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T14:16:41,213 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-12T14:16:41,214 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1290 sec 2024-11-12T14:16:41,218 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ea61e7c1d124f78e4a5241afdaab254 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T14:16:41,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=A 2024-11-12T14:16:41,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:41,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=B 2024-11-12T14:16:41,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:41,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=C 2024-11-12T14:16:41,219 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:41,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:41,252 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 3.2110 sec 2024-11-12T14:16:41,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/ea995fa4ab334484805f29697ee45978 is 50, key is test_row_0/A:col10/1731420999873/Put/seqid=0 2024-11-12T14:16:41,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741843_1019 (size=14341) 2024-11-12T14:16:41,370 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-12T14:16:41,445 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:41,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421061415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:41,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:41,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:41,446 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:41,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:41,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421061422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:41,453 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:41,453 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:41,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421061425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:41,456 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:41,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:41,457 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:41,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:41,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421061552, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:41,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:41,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:41,569 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:41,581 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:41,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421061564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:41,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:41,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421061564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:41,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:41,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:41,587 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:41,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:41,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:41,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:41,736 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/ea995fa4ab334484805f29697ee45978 2024-11-12T14:16:41,786 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421061780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:41,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:41,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:41,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:41,792 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:41,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421061790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:41,808 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:41,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:41,809 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:41,819 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/06eac540edb0487091891ee6f3d5846f is 50, key is test_row_0/B:col10/1731420999873/Put/seqid=0 2024-11-12T14:16:41,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:41,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421061810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:41,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:41,843 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:41,844 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:41,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741844_1020 (size=12001) 2024-11-12T14:16:41,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/06eac540edb0487091891ee6f3d5846f 2024-11-12T14:16:41,975 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/74fafa2273084d7c9adfca7a4b8efa30 is 50, key is test_row_0/C:col10/1731420999873/Put/seqid=0 2024-11-12T14:16:42,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741845_1021 (size=12001) 2024-11-12T14:16:42,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:42,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421062090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:42,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:42,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:42,109 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:42,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:42,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421062120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:42,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:42,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:42,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:42,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:42,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421062150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:42,176 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:42,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:42,177 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:42,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-12T14:16:42,201 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:16:42,201 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:16:42,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-12T14:16:42,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:42,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421062190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:42,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T14:16:42,205 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:42,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421062200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:42,210 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_2' in 6ea61e7c1d124f78e4a5241afdaab254 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2329 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:16:42,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:42,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:42,211 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:42,209 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:16:42,213 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:16:42,215 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:16:42,210 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_2' in 6ea61e7c1d124f78e4a5241afdaab254 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2333 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:16:42,217 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:42,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:42,218 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:42,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T14:16:42,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T14:16:42,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:42,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:42,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:42,374 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:42,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:42,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:42,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=61 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/74fafa2273084d7c9adfca7a4b8efa30 2024-11-12T14:16:42,505 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/ea995fa4ab334484805f29697ee45978 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/ea995fa4ab334484805f29697ee45978 2024-11-12T14:16:42,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T14:16:42,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T14:16:42,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:42,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:42,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:42,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:42,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:42,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:42,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/ea995fa4ab334484805f29697ee45978, entries=200, sequenceid=61, filesize=14.0 K 2024-11-12T14:16:42,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/06eac540edb0487091891ee6f3d5846f as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/06eac540edb0487091891ee6f3d5846f 2024-11-12T14:16:42,619 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/06eac540edb0487091891ee6f3d5846f, entries=150, sequenceid=61, filesize=11.7 K 2024-11-12T14:16:42,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/74fafa2273084d7c9adfca7a4b8efa30 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/74fafa2273084d7c9adfca7a4b8efa30 2024-11-12T14:16:42,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/74fafa2273084d7c9adfca7a4b8efa30, entries=150, sequenceid=61, filesize=11.7 K 2024-11-12T14:16:42,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 6ea61e7c1d124f78e4a5241afdaab254 in 1427ms, sequenceid=61, compaction requested=true 2024-11-12T14:16:42,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:42,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:42,671 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ea61e7c1d124f78e4a5241afdaab254:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:16:42,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:16:42,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ea61e7c1d124f78e4a5241afdaab254:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:16:42,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:16:42,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ea61e7c1d124f78e4a5241afdaab254:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:16:42,674 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T14:16:42,674 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:16:42,674 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ea61e7c1d124f78e4a5241afdaab254 3/3 column families, dataSize=114.05 KB heapSize=339.66 KB 2024-11-12T14:16:42,676 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:16:42,679 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:16:42,682 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:16:42,682 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 6ea61e7c1d124f78e4a5241afdaab254/A is initiating minor compaction (all files) 2024-11-12T14:16:42,682 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 6ea61e7c1d124f78e4a5241afdaab254/B is initiating minor compaction (all files) 2024-11-12T14:16:42,682 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ea61e7c1d124f78e4a5241afdaab254/A in TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:42,682 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ea61e7c1d124f78e4a5241afdaab254/B in TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:42,683 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/44505f337484442da4f854bbab0a652b, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/b7ddf445cfc04660882023922d42e0d7, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/06eac540edb0487091891ee6f3d5846f] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp, totalSize=35.2 K 2024-11-12T14:16:42,683 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/08feba09c95c43b484e9b2d0ab6fb503, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/bae66fce00154e688a47fb8355495de9, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/ea995fa4ab334484805f29697ee45978] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp, totalSize=39.7 K 2024-11-12T14:16:42,685 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 44505f337484442da4f854bbab0a652b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731420998119 2024-11-12T14:16:42,686 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting b7ddf445cfc04660882023922d42e0d7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731420999005 2024-11-12T14:16:42,687 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 06eac540edb0487091891ee6f3d5846f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731420999848 2024-11-12T14:16:42,692 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731421001378, max timestamp=1731421002663Num uniques 859; 2024-11-12T14:16:42,692 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T14:16:42,693 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 08feba09c95c43b484e9b2d0ab6fb503, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731420998119 2024-11-12T14:16:42,697 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting bae66fce00154e688a47fb8355495de9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731420999005 2024-11-12T14:16:42,699 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=A 2024-11-12T14:16:42,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:16:42,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=B 2024-11-12T14:16:42,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:16:42,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=C 2024-11-12T14:16:42,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:16:42,704 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T14:16:42,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:42,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:42,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:42,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:42,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:42,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:42,719 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea995fa4ab334484805f29697ee45978, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731420999848 2024-11-12T14:16:42,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/2c67e6f5e065492f8e61dbf29f7ccfed is 50, key is test_row_0/A:col10/1731421002663/Put/seqid=0 2024-11-12T14:16:42,812 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ea61e7c1d124f78e4a5241afdaab254#B#compaction#10 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:16:42,813 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/760c9b2620234350b0efc51a32e3e801 is 50, key is test_row_0/B:col10/1731420999873/Put/seqid=0 2024-11-12T14:16:42,829 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ea61e7c1d124f78e4a5241afdaab254#A#compaction#11 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:16:42,830 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/89690dae6c5d4aebabbfb9d8e7fa753b is 50, key is test_row_0/A:col10/1731420999873/Put/seqid=0 2024-11-12T14:16:42,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T14:16:42,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741846_1022 (size=14341) 2024-11-12T14:16:42,855 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/2c67e6f5e065492f8e61dbf29f7ccfed 2024-11-12T14:16:42,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T14:16:42,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:42,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:42,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:42,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:42,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:42,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:42,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741847_1023 (size=12104) 2024-11-12T14:16:42,948 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/760c9b2620234350b0efc51a32e3e801 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/760c9b2620234350b0efc51a32e3e801 2024-11-12T14:16:42,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/8a39f03e2d8f48a3beaa8c857f3dfcaa is 50, key is test_row_0/B:col10/1731421002663/Put/seqid=0 2024-11-12T14:16:42,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741848_1024 (size=12104) 2024-11-12T14:16:43,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741849_1025 (size=12001) 2024-11-12T14:16:43,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/8a39f03e2d8f48a3beaa8c857f3dfcaa 2024-11-12T14:16:43,019 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T14:16:43,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:43,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:43,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:43,021 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:43,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:43,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:43,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:43,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421063003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:43,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:43,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:43,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:43,035 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:43,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421063011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:43,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:43,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:43,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:43,059 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ea61e7c1d124f78e4a5241afdaab254/B of 6ea61e7c1d124f78e4a5241afdaab254 into 760c9b2620234350b0efc51a32e3e801(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:16:43,059 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:43,064 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:43,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421063031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:43,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:43,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:43,066 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:43,066 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., storeName=6ea61e7c1d124f78e4a5241afdaab254/B, priority=13, startTime=1731421002674; duration=0sec 2024-11-12T14:16:43,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/c997088aa181483eba1eaa405ebc8fd0 is 50, key is test_row_0/C:col10/1731421002663/Put/seqid=0 2024-11-12T14:16:43,077 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:16:43,078 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ea61e7c1d124f78e4a5241afdaab254:B 2024-11-12T14:16:43,078 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:16:43,086 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:16:43,090 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 6ea61e7c1d124f78e4a5241afdaab254/C is initiating minor compaction (all files) 2024-11-12T14:16:43,091 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ea61e7c1d124f78e4a5241afdaab254/C in TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:43,091 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/ae7f1e5c1b90408cb4aa1f895ae2e9d4, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/8bcf3c54b8204ef6a8d9ffe1f140ec5a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/74fafa2273084d7c9adfca7a4b8efa30] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp, totalSize=35.2 K 2024-11-12T14:16:43,097 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting ae7f1e5c1b90408cb4aa1f895ae2e9d4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731420998119 2024-11-12T14:16:43,099 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bcf3c54b8204ef6a8d9ffe1f140ec5a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731420999005 2024-11-12T14:16:43,108 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 74fafa2273084d7c9adfca7a4b8efa30, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731420999848 2024-11-12T14:16:43,121 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741850_1026 (size=12001) 2024-11-12T14:16:43,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=82 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/c997088aa181483eba1eaa405ebc8fd0 2024-11-12T14:16:43,154 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:43,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:43,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421063149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:43,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:43,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:43,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:43,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/2c67e6f5e065492f8e61dbf29f7ccfed as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/2c67e6f5e065492f8e61dbf29f7ccfed 2024-11-12T14:16:43,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421063140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:43,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T14:16:43,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:43,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:43,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:43,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:43,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:43,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:43,179 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:43,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:43,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:43,187 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ea61e7c1d124f78e4a5241afdaab254#C#compaction#14 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:16:43,188 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/12a4d09d0c1e43d2b02b0b0171850b7d is 50, key is test_row_0/C:col10/1731420999873/Put/seqid=0 2024-11-12T14:16:43,197 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/2c67e6f5e065492f8e61dbf29f7ccfed, entries=200, sequenceid=82, filesize=14.0 K 2024-11-12T14:16:43,206 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/8a39f03e2d8f48a3beaa8c857f3dfcaa as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/8a39f03e2d8f48a3beaa8c857f3dfcaa 2024-11-12T14:16:43,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:43,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421063170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:43,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:43,213 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:43,214 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:43,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/8a39f03e2d8f48a3beaa8c857f3dfcaa, entries=150, sequenceid=82, filesize=11.7 K 2024-11-12T14:16:43,222 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/c997088aa181483eba1eaa405ebc8fd0 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/c997088aa181483eba1eaa405ebc8fd0 2024-11-12T14:16:43,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/c997088aa181483eba1eaa405ebc8fd0, entries=150, sequenceid=82, filesize=11.7 K 2024-11-12T14:16:43,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741851_1027 (size=12104) 2024-11-12T14:16:43,257 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~89.45 KB/91600, heapSize ~262.06 KB/268352, currentSize=93.93 KB/96180 for 6ea61e7c1d124f78e4a5241afdaab254 in 583ms, sequenceid=82, compaction requested=false 2024-11-12T14:16:43,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:43,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T14:16:43,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-12T14:16:43,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:43,345 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 6ea61e7c1d124f78e4a5241afdaab254 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T14:16:43,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=A 2024-11-12T14:16:43,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:43,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=B 2024-11-12T14:16:43,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:43,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=C 2024-11-12T14:16:43,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:43,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/6c70741c43f94833a3f8c30a0cd12564 is 50, key is test_row_0/A:col10/1731421002988/Put/seqid=0 2024-11-12T14:16:43,393 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:43,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:43,404 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/89690dae6c5d4aebabbfb9d8e7fa753b as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/89690dae6c5d4aebabbfb9d8e7fa753b 2024-11-12T14:16:43,425 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ea61e7c1d124f78e4a5241afdaab254/A of 6ea61e7c1d124f78e4a5241afdaab254 into 89690dae6c5d4aebabbfb9d8e7fa753b(size=11.8 K), total size for store is 25.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:16:43,425 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:43,425 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., storeName=6ea61e7c1d124f78e4a5241afdaab254/A, priority=13, startTime=1731421002646; duration=0sec 2024-11-12T14:16:43,425 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:16:43,426 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ea61e7c1d124f78e4a5241afdaab254:A 2024-11-12T14:16:43,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741852_1028 (size=12001) 2024-11-12T14:16:43,449 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/6c70741c43f94833a3f8c30a0cd12564 2024-11-12T14:16:43,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/fba519f9cffa489595ce24b603522709 is 50, key is test_row_0/B:col10/1731421002988/Put/seqid=0 2024-11-12T14:16:43,547 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741853_1029 (size=12001) 2024-11-12T14:16:43,549 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/fba519f9cffa489595ce24b603522709 2024-11-12T14:16:43,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:43,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421063529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:43,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:43,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:43,560 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:43,578 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:43,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421063553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:43,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:43,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:43,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:43,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:43,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421063556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:43,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:43,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:43,580 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:43,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/7e495ffce5644be8a35d47b5bd4507b7 is 50, key is test_row_0/C:col10/1731421002988/Put/seqid=0 2024-11-12T14:16:43,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741854_1030 (size=12001) 2024-11-12T14:16:43,666 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=100 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/7e495ffce5644be8a35d47b5bd4507b7 2024-11-12T14:16:43,684 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/12a4d09d0c1e43d2b02b0b0171850b7d as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/12a4d09d0c1e43d2b02b0b0171850b7d 2024-11-12T14:16:43,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/6c70741c43f94833a3f8c30a0cd12564 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/6c70741c43f94833a3f8c30a0cd12564 2024-11-12T14:16:43,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:43,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421063676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:43,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:43,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:43,695 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:43,716 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ea61e7c1d124f78e4a5241afdaab254/C of 6ea61e7c1d124f78e4a5241afdaab254 into 12a4d09d0c1e43d2b02b0b0171850b7d(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:16:43,717 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:43,717 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., storeName=6ea61e7c1d124f78e4a5241afdaab254/C, priority=13, startTime=1731421002674; duration=0sec 2024-11-12T14:16:43,717 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:16:43,717 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ea61e7c1d124f78e4a5241afdaab254:C 2024-11-12T14:16:43,721 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/6c70741c43f94833a3f8c30a0cd12564, entries=150, sequenceid=100, filesize=11.7 K 2024-11-12T14:16:43,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:43,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/fba519f9cffa489595ce24b603522709 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/fba519f9cffa489595ce24b603522709 2024-11-12T14:16:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421063691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:43,725 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:43,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:43,726 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:43,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:43,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421063692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:43,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:43,758 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/fba519f9cffa489595ce24b603522709, entries=150, sequenceid=100, filesize=11.7 K 2024-11-12T14:16:43,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/7e495ffce5644be8a35d47b5bd4507b7 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/7e495ffce5644be8a35d47b5bd4507b7 2024-11-12T14:16:43,788 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/7e495ffce5644be8a35d47b5bd4507b7, entries=150, sequenceid=100, filesize=11.7 K 2024-11-12T14:16:43,791 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 6ea61e7c1d124f78e4a5241afdaab254 in 446ms, sequenceid=100, compaction requested=true 2024-11-12T14:16:43,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:43,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:43,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-12T14:16:43,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-12T14:16:43,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-12T14:16:43,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5870 sec 2024-11-12T14:16:43,811 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.6070 sec 2024-11-12T14:16:43,913 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ea61e7c1d124f78e4a5241afdaab254 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T14:16:43,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=A 2024-11-12T14:16:43,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:43,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=B 2024-11-12T14:16:43,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:43,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=C 2024-11-12T14:16:43,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:43,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:43,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/461922825eaf46aca28b9b7f40945202 is 50, key is test_row_0/A:col10/1731421003547/Put/seqid=0 2024-11-12T14:16:43,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741855_1031 (size=12001) 2024-11-12T14:16:44,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:44,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421064049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:44,076 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:44,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:44,077 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:44,086 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:44,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421064075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:44,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:44,087 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:44,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:44,087 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:44,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421064076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:44,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:44,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:44,088 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:44,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:44,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421064180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:44,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:44,206 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:44,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421064200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:44,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:44,207 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:44,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:44,207 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:44,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421064200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:44,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:44,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:44,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:44,245 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:44,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421064232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:44,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:44,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421064232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:44,246 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_2' in 6ea61e7c1d124f78e4a5241afdaab254 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4371 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:16:44,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:44,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:44,247 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:44,247 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_2' in 6ea61e7c1d124f78e4a5241afdaab254 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4369 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:16:44,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:44,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:44,250 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:44,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-12T14:16:44,351 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:16:44,351 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:16:44,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-12T14:16:44,365 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:16:44,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T14:16:44,376 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:16:44,377 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:16:44,378 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/461922825eaf46aca28b9b7f40945202 2024-11-12T14:16:44,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:44,427 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:44,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421064400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:44,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:44,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421064413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:44,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:44,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421064420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:44,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:44,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:44,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:44,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:44,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:44,431 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:44,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/8a19c7d41efc44c593cfce707c3a1fb9 is 50, key is test_row_0/B:col10/1731421003547/Put/seqid=0 2024-11-12T14:16:44,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T14:16:44,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741856_1032 (size=12001) 2024-11-12T14:16:44,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/8a19c7d41efc44c593cfce707c3a1fb9 2024-11-12T14:16:44,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-12T14:16:44,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:44,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:44,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:44,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:44,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:44,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:44,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/8b2517ec1ce74432b0c6f376876e120a is 50, key is test_row_0/C:col10/1731421003547/Put/seqid=0 2024-11-12T14:16:44,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741857_1033 (size=12001) 2024-11-12T14:16:44,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/8b2517ec1ce74432b0c6f376876e120a 2024-11-12T14:16:44,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T14:16:44,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-12T14:16:44,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:44,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:44,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:44,697 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:44,697 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:44,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:44,713 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/461922825eaf46aca28b9b7f40945202 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/461922825eaf46aca28b9b7f40945202 2024-11-12T14:16:44,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:44,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421064750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:44,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:44,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:44,754 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:44,759 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:44,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421064753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:44,764 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:44,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:44,765 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:44,770 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:44,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421064753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:44,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:44,772 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:44,773 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:44,773 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/461922825eaf46aca28b9b7f40945202, entries=150, sequenceid=119, filesize=11.7 K 2024-11-12T14:16:44,782 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/8a19c7d41efc44c593cfce707c3a1fb9 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/8a19c7d41efc44c593cfce707c3a1fb9 2024-11-12T14:16:44,827 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/8a19c7d41efc44c593cfce707c3a1fb9, entries=150, sequenceid=119, filesize=11.7 K 2024-11-12T14:16:44,832 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/8b2517ec1ce74432b0c6f376876e120a as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/8b2517ec1ce74432b0c6f376876e120a 2024-11-12T14:16:44,866 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-12T14:16:44,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:44,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:44,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:44,867 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:44,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:44,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:44,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/8b2517ec1ce74432b0c6f376876e120a, entries=150, sequenceid=119, filesize=11.7 K 2024-11-12T14:16:44,911 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=87.22 KB/89310 for 6ea61e7c1d124f78e4a5241afdaab254 in 999ms, sequenceid=119, compaction requested=true 2024-11-12T14:16:44,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:44,912 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:16:44,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ea61e7c1d124f78e4a5241afdaab254:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:16:44,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:16:44,913 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:16:44,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ea61e7c1d124f78e4a5241afdaab254:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:16:44,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:16:44,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ea61e7c1d124f78e4a5241afdaab254:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:16:44,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:16:44,924 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50447 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:16:44,924 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:16:44,924 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 6ea61e7c1d124f78e4a5241afdaab254/A is initiating minor compaction (all files) 2024-11-12T14:16:44,924 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 6ea61e7c1d124f78e4a5241afdaab254/B is initiating minor compaction (all files) 2024-11-12T14:16:44,924 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ea61e7c1d124f78e4a5241afdaab254/A in TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:44,924 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ea61e7c1d124f78e4a5241afdaab254/B in TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:44,925 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/89690dae6c5d4aebabbfb9d8e7fa753b, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/2c67e6f5e065492f8e61dbf29f7ccfed, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/6c70741c43f94833a3f8c30a0cd12564, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/461922825eaf46aca28b9b7f40945202] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp, totalSize=49.3 K 2024-11-12T14:16:44,925 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/760c9b2620234350b0efc51a32e3e801, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/8a39f03e2d8f48a3beaa8c857f3dfcaa, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/fba519f9cffa489595ce24b603522709, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/8a19c7d41efc44c593cfce707c3a1fb9] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp, totalSize=47.0 K 2024-11-12T14:16:44,931 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 760c9b2620234350b0efc51a32e3e801, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731420999848 2024-11-12T14:16:44,936 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89690dae6c5d4aebabbfb9d8e7fa753b, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731420999848 2024-11-12T14:16:44,937 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a39f03e2d8f48a3beaa8c857f3dfcaa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731421001378 2024-11-12T14:16:44,948 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting fba519f9cffa489595ce24b603522709, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731421002952 2024-11-12T14:16:44,948 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c67e6f5e065492f8e61dbf29f7ccfed, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731421001378 2024-11-12T14:16:44,949 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c70741c43f94833a3f8c30a0cd12564, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731421002952 2024-11-12T14:16:44,950 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a19c7d41efc44c593cfce707c3a1fb9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731421003511 2024-11-12T14:16:44,950 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 461922825eaf46aca28b9b7f40945202, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731421003511 2024-11-12T14:16:45,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T14:16:45,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-12T14:16:45,035 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ea61e7c1d124f78e4a5241afdaab254#A#compaction#22 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:16:45,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:45,040 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 6ea61e7c1d124f78e4a5241afdaab254 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-12T14:16:45,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=A 2024-11-12T14:16:45,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:45,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=B 2024-11-12T14:16:45,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:45,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=C 2024-11-12T14:16:45,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:45,042 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/5422e64db8244b6fa12dd510e8660d4c is 50, key is test_row_0/A:col10/1731421003547/Put/seqid=0 2024-11-12T14:16:45,050 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ea61e7c1d124f78e4a5241afdaab254#B#compaction#21 average throughput is 0.34 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:16:45,051 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/c3cda5212db34baa9fffb5dcef511632 is 50, key is test_row_0/B:col10/1731421003547/Put/seqid=0 2024-11-12T14:16:45,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741858_1034 (size=12241) 2024-11-12T14:16:45,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741859_1035 (size=12241) 2024-11-12T14:16:45,139 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/5d0e9dd38ea6449585fc49882de5d331 is 50, key is test_row_0/A:col10/1731421004059/Put/seqid=0 2024-11-12T14:16:45,145 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/5422e64db8244b6fa12dd510e8660d4c as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/5422e64db8244b6fa12dd510e8660d4c 2024-11-12T14:16:45,170 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ea61e7c1d124f78e4a5241afdaab254/A of 6ea61e7c1d124f78e4a5241afdaab254 into 5422e64db8244b6fa12dd510e8660d4c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:16:45,170 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:45,170 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., storeName=6ea61e7c1d124f78e4a5241afdaab254/A, priority=12, startTime=1731421004912; duration=0sec 2024-11-12T14:16:45,171 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:16:45,171 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ea61e7c1d124f78e4a5241afdaab254:A 2024-11-12T14:16:45,171 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:16:45,176 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:16:45,176 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 6ea61e7c1d124f78e4a5241afdaab254/C is initiating minor compaction (all files) 2024-11-12T14:16:45,176 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ea61e7c1d124f78e4a5241afdaab254/C in TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:45,177 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/12a4d09d0c1e43d2b02b0b0171850b7d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/c997088aa181483eba1eaa405ebc8fd0, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/7e495ffce5644be8a35d47b5bd4507b7, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/8b2517ec1ce74432b0c6f376876e120a] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp, totalSize=47.0 K 2024-11-12T14:16:45,179 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12a4d09d0c1e43d2b02b0b0171850b7d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=61, earliestPutTs=1731420999848 2024-11-12T14:16:45,183 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting c997088aa181483eba1eaa405ebc8fd0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=82, earliestPutTs=1731421001378 2024-11-12T14:16:45,185 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e495ffce5644be8a35d47b5bd4507b7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=100, earliestPutTs=1731421002952 2024-11-12T14:16:45,186 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b2517ec1ce74432b0c6f376876e120a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731421003511 2024-11-12T14:16:45,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741860_1036 (size=12151) 2024-11-12T14:16:45,193 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/5d0e9dd38ea6449585fc49882de5d331 2024-11-12T14:16:45,232 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ea61e7c1d124f78e4a5241afdaab254#C#compaction#24 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:16:45,233 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/0a7806d6a19243d3ab57c6240bacd0dc is 50, key is test_row_0/C:col10/1731421003547/Put/seqid=0 2024-11-12T14:16:45,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/f1c87643e3db4ff58b211ca4ee173984 is 50, key is test_row_0/B:col10/1731421004059/Put/seqid=0 2024-11-12T14:16:45,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:45,269 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:45,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741861_1037 (size=12241) 2024-11-12T14:16:45,287 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/0a7806d6a19243d3ab57c6240bacd0dc as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/0a7806d6a19243d3ab57c6240bacd0dc 2024-11-12T14:16:45,315 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ea61e7c1d124f78e4a5241afdaab254/C of 6ea61e7c1d124f78e4a5241afdaab254 into 0a7806d6a19243d3ab57c6240bacd0dc(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:16:45,315 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:45,315 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., storeName=6ea61e7c1d124f78e4a5241afdaab254/C, priority=12, startTime=1731421004913; duration=0sec 2024-11-12T14:16:45,315 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:16:45,315 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ea61e7c1d124f78e4a5241afdaab254:C 2024-11-12T14:16:45,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741862_1038 (size=12151) 2024-11-12T14:16:45,356 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/f1c87643e3db4ff58b211ca4ee173984 2024-11-12T14:16:45,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:45,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421065362, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:45,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:45,375 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:45,376 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:45,388 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:45,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421065372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:45,394 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:45,405 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:45,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421065376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:45,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:45,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:45,406 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:45,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/a46b881be4224f7995b73d9ad2276353 is 50, key is test_row_0/C:col10/1731421004059/Put/seqid=0 2024-11-12T14:16:45,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741863_1039 (size=12151) 2024-11-12T14:16:45,496 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:45,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421065479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:45,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:45,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:45,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:45,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:45,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421065500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:45,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:45,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:45,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:45,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T14:16:45,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:45,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421065513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:45,536 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:45,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:45,537 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:45,546 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/c3cda5212db34baa9fffb5dcef511632 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/c3cda5212db34baa9fffb5dcef511632 2024-11-12T14:16:45,570 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ea61e7c1d124f78e4a5241afdaab254/B of 6ea61e7c1d124f78e4a5241afdaab254 into c3cda5212db34baa9fffb5dcef511632(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:16:45,570 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:45,570 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., storeName=6ea61e7c1d124f78e4a5241afdaab254/B, priority=12, startTime=1731421004912; duration=0sec 2024-11-12T14:16:45,570 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:16:45,570 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ea61e7c1d124f78e4a5241afdaab254:B 2024-11-12T14:16:45,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:45,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421065719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:45,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:45,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:45,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:45,740 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:45,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421065709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:45,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:45,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:45,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:45,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:45,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421065756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:45,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:45,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:45,769 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:45,888 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/a46b881be4224f7995b73d9ad2276353 2024-11-12T14:16:45,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/5d0e9dd38ea6449585fc49882de5d331 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/5d0e9dd38ea6449585fc49882de5d331 2024-11-12T14:16:45,936 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/5d0e9dd38ea6449585fc49882de5d331, entries=150, sequenceid=135, filesize=11.9 K 2024-11-12T14:16:45,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/f1c87643e3db4ff58b211ca4ee173984 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/f1c87643e3db4ff58b211ca4ee173984 2024-11-12T14:16:45,959 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/f1c87643e3db4ff58b211ca4ee173984, entries=150, sequenceid=135, filesize=11.9 K 2024-11-12T14:16:45,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/a46b881be4224f7995b73d9ad2276353 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/a46b881be4224f7995b73d9ad2276353 2024-11-12T14:16:45,980 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/a46b881be4224f7995b73d9ad2276353, entries=150, sequenceid=135, filesize=11.9 K 2024-11-12T14:16:45,982 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 6ea61e7c1d124f78e4a5241afdaab254 in 942ms, sequenceid=135, compaction requested=false 2024-11-12T14:16:45,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:45,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:45,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-12T14:16:45,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-12T14:16:45,999 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-12T14:16:45,999 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6140 sec 2024-11-12T14:16:46,005 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.6510 sec 2024-11-12T14:16:46,056 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ea61e7c1d124f78e4a5241afdaab254 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T14:16:46,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:46,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=A 2024-11-12T14:16:46,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:46,059 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=B 2024-11-12T14:16:46,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:46,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=C 2024-11-12T14:16:46,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:46,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/b8baffd54d25404f914dd772c83182cc is 50, key is test_row_0/A:col10/1731421006054/Put/seqid=0 2024-11-12T14:16:46,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:46,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421066119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:46,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:46,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:46,141 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:46,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:46,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421066121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:46,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:46,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421066140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:46,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:46,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:46,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:46,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741864_1040 (size=14541) 2024-11-12T14:16:46,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:46,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:46,154 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:46,256 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:46,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421066249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:46,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:46,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421066259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:46,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:46,266 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:46,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:46,266 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:46,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421066259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:46,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:46,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:46,267 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:46,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:46,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:46,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:46,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:46,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421066472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:46,476 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:46,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:46,477 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:46,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:46,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421066480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:46,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:46,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:46,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:46,485 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:46,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421066480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:46,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:46,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:46,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:46,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-12T14:16:46,540 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:16:46,540 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:16:46,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-12T14:16:46,544 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:16:46,546 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:16:46,546 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:16:46,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T14:16:46,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/b8baffd54d25404f914dd772c83182cc 2024-11-12T14:16:46,615 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/6711321620c34778ac87e055f42c589f is 50, key is test_row_0/B:col10/1731421006054/Put/seqid=0 2024-11-12T14:16:46,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T14:16:46,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741865_1041 (size=12151) 2024-11-12T14:16:46,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T14:16:46,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:46,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:46,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:46,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:46,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:46,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:46,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:46,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421066844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:46,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:46,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:46,848 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:46,856 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:46,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421066848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:46,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:46,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:46,858 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:46,861 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:46,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421066848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:46,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:46,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:46,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:46,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T14:16:46,870 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T14:16:46,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:46,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:46,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:46,873 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:46,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:46,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:47,034 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T14:16:47,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:47,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:47,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:47,034 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:47,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:47,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:47,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/6711321620c34778ac87e055f42c589f 2024-11-12T14:16:47,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/0b27d952afda42a2bdcab96d2d0ba1d7 is 50, key is test_row_0/C:col10/1731421006054/Put/seqid=0 2024-11-12T14:16:47,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741866_1042 (size=12151) 2024-11-12T14:16:47,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T14:16:47,197 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T14:16:47,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:47,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:47,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:47,197 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:47,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:47,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:47,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T14:16:47,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:47,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:47,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:47,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:47,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:47,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:47,375 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:47,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421067364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:47,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:47,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:47,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:47,386 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:47,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421067373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:47,388 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:47,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421067373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:47,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:47,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:47,389 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:47,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:47,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:47,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:47,529 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T14:16:47,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:47,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:47,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:47,529 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:47,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:47,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:47,590 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/0b27d952afda42a2bdcab96d2d0ba1d7 2024-11-12T14:16:47,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/b8baffd54d25404f914dd772c83182cc as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/b8baffd54d25404f914dd772c83182cc 2024-11-12T14:16:47,627 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/b8baffd54d25404f914dd772c83182cc, entries=200, sequenceid=156, filesize=14.2 K 2024-11-12T14:16:47,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/6711321620c34778ac87e055f42c589f as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/6711321620c34778ac87e055f42c589f 2024-11-12T14:16:47,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/6711321620c34778ac87e055f42c589f, entries=150, sequenceid=156, filesize=11.9 K 2024-11-12T14:16:47,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/0b27d952afda42a2bdcab96d2d0ba1d7 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/0b27d952afda42a2bdcab96d2d0ba1d7 2024-11-12T14:16:47,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/0b27d952afda42a2bdcab96d2d0ba1d7, entries=150, sequenceid=156, filesize=11.9 K 2024-11-12T14:16:47,665 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 6ea61e7c1d124f78e4a5241afdaab254 in 1609ms, sequenceid=156, compaction requested=true 2024-11-12T14:16:47,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:47,665 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:16:47,668 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38933 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:16:47,668 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 6ea61e7c1d124f78e4a5241afdaab254/A is initiating minor compaction (all files) 2024-11-12T14:16:47,668 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ea61e7c1d124f78e4a5241afdaab254/A in TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:47,668 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/5422e64db8244b6fa12dd510e8660d4c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/5d0e9dd38ea6449585fc49882de5d331, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/b8baffd54d25404f914dd772c83182cc] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp, totalSize=38.0 K 2024-11-12T14:16:47,669 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5422e64db8244b6fa12dd510e8660d4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731421003511 2024-11-12T14:16:47,670 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d0e9dd38ea6449585fc49882de5d331, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731421004015 2024-11-12T14:16:47,670 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8baffd54d25404f914dd772c83182cc, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731421005343 2024-11-12T14:16:47,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ea61e7c1d124f78e4a5241afdaab254:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:16:47,681 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:16:47,681 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:16:47,683 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:16:47,683 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 6ea61e7c1d124f78e4a5241afdaab254/B is initiating minor compaction (all files) 2024-11-12T14:16:47,683 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ea61e7c1d124f78e4a5241afdaab254/B in TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:47,683 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/c3cda5212db34baa9fffb5dcef511632, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/f1c87643e3db4ff58b211ca4ee173984, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/6711321620c34778ac87e055f42c589f] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp, totalSize=35.7 K 2024-11-12T14:16:47,684 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting c3cda5212db34baa9fffb5dcef511632, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731421003511 2024-11-12T14:16:47,685 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting f1c87643e3db4ff58b211ca4ee173984, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731421004015 2024-11-12T14:16:47,685 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ea61e7c1d124f78e4a5241afdaab254:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:16:47,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:16:47,686 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 6711321620c34778ac87e055f42c589f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731421005343 2024-11-12T14:16:47,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ea61e7c1d124f78e4a5241afdaab254:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:16:47,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:16:47,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-12T14:16:47,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:47,690 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 6ea61e7c1d124f78e4a5241afdaab254 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T14:16:47,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=A 2024-11-12T14:16:47,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:47,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=B 2024-11-12T14:16:47,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:47,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=C 2024-11-12T14:16:47,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:47,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T14:16:47,702 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ea61e7c1d124f78e4a5241afdaab254#A#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:16:47,703 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/9961ff4a421a48fcaf6673d6350b5664 is 50, key is test_row_0/A:col10/1731421006054/Put/seqid=0 2024-11-12T14:16:47,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/503976c02e694e2182bab60d8505a9d1 is 50, key is test_row_0/A:col10/1731421006105/Put/seqid=0 2024-11-12T14:16:47,743 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ea61e7c1d124f78e4a5241afdaab254#B#compaction#32 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:16:47,744 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/67d7280b788142d1acff8c921423ed7e is 50, key is test_row_0/B:col10/1731421006054/Put/seqid=0 2024-11-12T14:16:47,822 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741868_1044 (size=12151) 2024-11-12T14:16:47,824 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/503976c02e694e2182bab60d8505a9d1 2024-11-12T14:16:47,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741869_1045 (size=12493) 2024-11-12T14:16:47,838 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741867_1043 (size=12493) 2024-11-12T14:16:47,892 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/67d7280b788142d1acff8c921423ed7e as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/67d7280b788142d1acff8c921423ed7e 2024-11-12T14:16:47,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/15729b4f1eef4a05b2494144594a77e7 is 50, key is test_row_0/B:col10/1731421006105/Put/seqid=0 2024-11-12T14:16:47,937 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/9961ff4a421a48fcaf6673d6350b5664 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/9961ff4a421a48fcaf6673d6350b5664 2024-11-12T14:16:47,953 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ea61e7c1d124f78e4a5241afdaab254/B of 6ea61e7c1d124f78e4a5241afdaab254 into 67d7280b788142d1acff8c921423ed7e(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:16:47,953 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:47,953 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., storeName=6ea61e7c1d124f78e4a5241afdaab254/B, priority=13, startTime=1731421007681; duration=0sec 2024-11-12T14:16:47,954 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:16:47,954 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ea61e7c1d124f78e4a5241afdaab254:B 2024-11-12T14:16:47,954 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:16:47,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741870_1046 (size=12151) 2024-11-12T14:16:47,967 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36543 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:16:47,967 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 6ea61e7c1d124f78e4a5241afdaab254/C is initiating minor compaction (all files) 2024-11-12T14:16:47,967 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ea61e7c1d124f78e4a5241afdaab254/C in TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:47,967 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/0a7806d6a19243d3ab57c6240bacd0dc, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/a46b881be4224f7995b73d9ad2276353, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/0b27d952afda42a2bdcab96d2d0ba1d7] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp, totalSize=35.7 K 2024-11-12T14:16:47,968 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a7806d6a19243d3ab57c6240bacd0dc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731421003511 2024-11-12T14:16:47,968 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/15729b4f1eef4a05b2494144594a77e7 2024-11-12T14:16:47,972 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting a46b881be4224f7995b73d9ad2276353, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731421004015 2024-11-12T14:16:47,973 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b27d952afda42a2bdcab96d2d0ba1d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731421005343 2024-11-12T14:16:48,001 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ea61e7c1d124f78e4a5241afdaab254/A of 6ea61e7c1d124f78e4a5241afdaab254 into 9961ff4a421a48fcaf6673d6350b5664(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:16:48,002 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:48,002 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., storeName=6ea61e7c1d124f78e4a5241afdaab254/A, priority=13, startTime=1731421007665; duration=0sec 2024-11-12T14:16:48,003 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:16:48,003 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ea61e7c1d124f78e4a5241afdaab254:A 2024-11-12T14:16:48,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/a037613c9dac4b6ab9fb675c81211c75 is 50, key is test_row_0/C:col10/1731421006105/Put/seqid=0 2024-11-12T14:16:48,056 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ea61e7c1d124f78e4a5241afdaab254#C#compaction#35 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:16:48,057 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/66beeb7f86404bcd97fe236f558678ee is 50, key is test_row_0/C:col10/1731421006054/Put/seqid=0 2024-11-12T14:16:48,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741871_1047 (size=12151) 2024-11-12T14:16:48,124 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/a037613c9dac4b6ab9fb675c81211c75 2024-11-12T14:16:48,141 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741872_1048 (size=12493) 2024-11-12T14:16:48,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/503976c02e694e2182bab60d8505a9d1 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/503976c02e694e2182bab60d8505a9d1 2024-11-12T14:16:48,162 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/66beeb7f86404bcd97fe236f558678ee as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/66beeb7f86404bcd97fe236f558678ee 2024-11-12T14:16:48,172 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/503976c02e694e2182bab60d8505a9d1, entries=150, sequenceid=170, filesize=11.9 K 2024-11-12T14:16:48,188 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ea61e7c1d124f78e4a5241afdaab254/C of 6ea61e7c1d124f78e4a5241afdaab254 into 66beeb7f86404bcd97fe236f558678ee(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:16:48,189 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:48,189 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., storeName=6ea61e7c1d124f78e4a5241afdaab254/C, priority=13, startTime=1731421007686; duration=0sec 2024-11-12T14:16:48,189 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:16:48,189 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ea61e7c1d124f78e4a5241afdaab254:C 2024-11-12T14:16:48,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/15729b4f1eef4a05b2494144594a77e7 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/15729b4f1eef4a05b2494144594a77e7 2024-11-12T14:16:48,246 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/15729b4f1eef4a05b2494144594a77e7, entries=150, sequenceid=170, filesize=11.9 K 2024-11-12T14:16:48,253 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/a037613c9dac4b6ab9fb675c81211c75 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/a037613c9dac4b6ab9fb675c81211c75 2024-11-12T14:16:48,274 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/a037613c9dac4b6ab9fb675c81211c75, entries=150, sequenceid=170, filesize=11.9 K 2024-11-12T14:16:48,275 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=0 B/0 for 6ea61e7c1d124f78e4a5241afdaab254 in 585ms, sequenceid=170, compaction requested=false 2024-11-12T14:16:48,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:48,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:48,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-12T14:16:48,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-12T14:16:48,284 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-12T14:16:48,284 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7340 sec 2024-11-12T14:16:48,289 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.7450 sec 2024-11-12T14:16:48,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:48,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ea61e7c1d124f78e4a5241afdaab254 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T14:16:48,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=A 2024-11-12T14:16:48,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:48,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=B 2024-11-12T14:16:48,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:48,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=C 2024-11-12T14:16:48,370 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:48,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/d21d5428f2f24845b6e4dd7fa22e1c73 is 50, key is test_row_0/A:col10/1731421008317/Put/seqid=0 2024-11-12T14:16:48,417 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741873_1049 (size=14541) 2024-11-12T14:16:48,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/d21d5428f2f24845b6e4dd7fa22e1c73 2024-11-12T14:16:48,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/374e05b3346940aeb1a34ce699c567e6 is 50, key is test_row_0/B:col10/1731421008317/Put/seqid=0 2024-11-12T14:16:48,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741874_1050 (size=12151) 2024-11-12T14:16:48,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/374e05b3346940aeb1a34ce699c567e6 2024-11-12T14:16:48,517 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:48,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421068471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:48,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421068471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:48,518 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:48,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421068488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:48,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:48,520 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:48,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:48,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:48,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:48,521 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:48,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:48,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:48,521 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:48,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731421008447, max timestamp=1731421008472Num uniques 859; 2024-11-12T14:16:48,530 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T14:16:48,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/d04fe8ccd68e4174a71711348ad65b71 is 50, key is test_row_0/C:col10/1731421008317/Put/seqid=0 2024-11-12T14:16:48,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:48,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421068574, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:48,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:48,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:48,587 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:48,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741875_1051 (size=12151) 2024-11-12T14:16:48,592 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/d04fe8ccd68e4174a71711348ad65b71 2024-11-12T14:16:48,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:48,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421068581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:48,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:48,598 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:48,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/d21d5428f2f24845b6e4dd7fa22e1c73 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/d21d5428f2f24845b6e4dd7fa22e1c73 2024-11-12T14:16:48,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/d21d5428f2f24845b6e4dd7fa22e1c73, entries=200, sequenceid=183, filesize=14.2 K 2024-11-12T14:16:48,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/374e05b3346940aeb1a34ce699c567e6 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/374e05b3346940aeb1a34ce699c567e6 2024-11-12T14:16:48,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:48,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421068631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:48,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:48,640 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:48,641 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:48,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:48,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421068634, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:48,642 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/374e05b3346940aeb1a34ce699c567e6, entries=150, sequenceid=183, filesize=11.9 K 2024-11-12T14:16:48,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:48,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421068636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:48,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:48,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:48,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:48,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/d04fe8ccd68e4174a71711348ad65b71 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/d04fe8ccd68e4174a71711348ad65b71 2024-11-12T14:16:48,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:48,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:48,644 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:48,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/d04fe8ccd68e4174a71711348ad65b71, entries=150, sequenceid=183, filesize=11.9 K 2024-11-12T14:16:48,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=129.71 KB/132820 for 6ea61e7c1d124f78e4a5241afdaab254 in 285ms, sequenceid=183, compaction requested=true 2024-11-12T14:16:48,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:48,655 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:16:48,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ea61e7c1d124f78e4a5241afdaab254:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:16:48,656 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:16:48,656 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:16:48,656 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:16:48,657 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 6ea61e7c1d124f78e4a5241afdaab254/A is initiating minor compaction (all files) 2024-11-12T14:16:48,657 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ea61e7c1d124f78e4a5241afdaab254/A in TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:48,657 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/9961ff4a421a48fcaf6673d6350b5664, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/503976c02e694e2182bab60d8505a9d1, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/d21d5428f2f24845b6e4dd7fa22e1c73] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp, totalSize=38.3 K 2024-11-12T14:16:48,658 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:16:48,658 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 6ea61e7c1d124f78e4a5241afdaab254/B is initiating minor compaction (all files) 2024-11-12T14:16:48,658 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ea61e7c1d124f78e4a5241afdaab254/B in TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:48,658 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/67d7280b788142d1acff8c921423ed7e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/15729b4f1eef4a05b2494144594a77e7, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/374e05b3346940aeb1a34ce699c567e6] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp, totalSize=35.9 K 2024-11-12T14:16:48,659 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9961ff4a421a48fcaf6673d6350b5664, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731421005343 2024-11-12T14:16:48,659 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 67d7280b788142d1acff8c921423ed7e, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731421005343 2024-11-12T14:16:48,660 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 503976c02e694e2182bab60d8505a9d1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731421006105 2024-11-12T14:16:48,660 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 15729b4f1eef4a05b2494144594a77e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731421006105 2024-11-12T14:16:48,661 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting d21d5428f2f24845b6e4dd7fa22e1c73, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731421008317 2024-11-12T14:16:48,661 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 374e05b3346940aeb1a34ce699c567e6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731421008317 2024-11-12T14:16:48,683 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ea61e7c1d124f78e4a5241afdaab254#B#compaction#39 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:16:48,684 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/435d7cfdb6474d18859d44263c84fac5 is 50, key is test_row_0/B:col10/1731421008317/Put/seqid=0 2024-11-12T14:16:48,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ea61e7c1d124f78e4a5241afdaab254:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:16:48,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:16:48,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ea61e7c1d124f78e4a5241afdaab254:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:16:48,684 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:16:48,688 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ea61e7c1d124f78e4a5241afdaab254#A#compaction#40 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:16:48,690 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/ccfc6d3beaf144679f45b7eaa984aded is 50, key is test_row_0/A:col10/1731421008317/Put/seqid=0 2024-11-12T14:16:48,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-12T14:16:48,701 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:16:48,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:48,701 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:16:48,703 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ea61e7c1d124f78e4a5241afdaab254 3/3 column families, dataSize=136.42 KB heapSize=401.56 KB 2024-11-12T14:16:48,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=A 2024-11-12T14:16:48,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:16:48,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=B 2024-11-12T14:16:48,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:16:48,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=C 2024-11-12T14:16:48,703 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:16:48,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-12T14:16:48,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T14:16:48,709 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:16:48,711 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:16:48,711 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:16:48,760 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/b754eeb71830426fb7f9e21aaa5712cf is 50, key is test_row_0/A:col10/1731421008447/Put/seqid=0 2024-11-12T14:16:48,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741877_1053 (size=12595) 2024-11-12T14:16:48,795 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/ccfc6d3beaf144679f45b7eaa984aded as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/ccfc6d3beaf144679f45b7eaa984aded 2024-11-12T14:16:48,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741876_1052 (size=12595) 2024-11-12T14:16:48,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T14:16:48,841 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741878_1054 (size=14541) 2024-11-12T14:16:48,844 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/435d7cfdb6474d18859d44263c84fac5 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/435d7cfdb6474d18859d44263c84fac5 2024-11-12T14:16:48,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/b754eeb71830426fb7f9e21aaa5712cf 2024-11-12T14:16:48,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-12T14:16:48,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:48,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:48,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:48,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:48,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:48,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:48,882 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ea61e7c1d124f78e4a5241afdaab254/A of 6ea61e7c1d124f78e4a5241afdaab254 into ccfc6d3beaf144679f45b7eaa984aded(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:16:48,882 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:48,882 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., storeName=6ea61e7c1d124f78e4a5241afdaab254/A, priority=13, startTime=1731421008655; duration=0sec 2024-11-12T14:16:48,883 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:16:48,883 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ea61e7c1d124f78e4a5241afdaab254:A 2024-11-12T14:16:48,883 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:16:48,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:48,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421068852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:48,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:48,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:48,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:48,897 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:16:48,897 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 6ea61e7c1d124f78e4a5241afdaab254/C is initiating minor compaction (all files) 2024-11-12T14:16:48,897 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ea61e7c1d124f78e4a5241afdaab254/C in TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:48,897 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/66beeb7f86404bcd97fe236f558678ee, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/a037613c9dac4b6ab9fb675c81211c75, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/d04fe8ccd68e4174a71711348ad65b71] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp, totalSize=35.9 K 2024-11-12T14:16:48,904 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ea61e7c1d124f78e4a5241afdaab254/B of 6ea61e7c1d124f78e4a5241afdaab254 into 435d7cfdb6474d18859d44263c84fac5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:16:48,904 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:48,905 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., storeName=6ea61e7c1d124f78e4a5241afdaab254/B, priority=13, startTime=1731421008656; duration=0sec 2024-11-12T14:16:48,905 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:16:48,905 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/9feee99b998747f5a286d5670f1323a2 is 50, key is test_row_0/B:col10/1731421008447/Put/seqid=0 2024-11-12T14:16:48,905 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ea61e7c1d124f78e4a5241afdaab254:B 2024-11-12T14:16:48,906 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66beeb7f86404bcd97fe236f558678ee, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731421005343 2024-11-12T14:16:48,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421068885, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:48,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:48,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:48,907 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:48,909 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting a037613c9dac4b6ab9fb675c81211c75, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731421006105 2024-11-12T14:16:48,911 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting d04fe8ccd68e4174a71711348ad65b71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731421008317 2024-11-12T14:16:48,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:48,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421068894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:48,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:48,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421068886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:48,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:48,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:48,913 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:48,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:48,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:48,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:48,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:48,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421068908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:48,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:48,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:48,933 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:48,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741879_1055 (size=12151) 2024-11-12T14:16:48,949 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/9feee99b998747f5a286d5670f1323a2 2024-11-12T14:16:48,959 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ea61e7c1d124f78e4a5241afdaab254#C#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:16:48,960 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/297ffb33b2964320b29c315ca171b6de is 50, key is test_row_0/C:col10/1731421008317/Put/seqid=0 2024-11-12T14:16:49,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741880_1056 (size=12595) 2024-11-12T14:16:49,011 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/d1244190df6840948e3a846e9e7dde13 is 50, key is test_row_0/C:col10/1731421008447/Put/seqid=0 2024-11-12T14:16:49,012 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:49,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421069010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:49,013 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:49,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:49,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T14:16:49,030 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-12T14:16:49,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:49,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:49,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:49,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:49,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:49,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:49,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421069020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:49,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:49,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:49,034 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:49,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:49,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:49,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421069040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:49,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:49,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:49,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421069040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:49,046 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:49,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:49,047 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:49,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741881_1057 (size=12151) 2024-11-12T14:16:49,192 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-12T14:16:49,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:49,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:49,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:49,196 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:49,197 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:49,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:49,219 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:49,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421069200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:49,220 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:49,224 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:49,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:49,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421069220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:49,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:49,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:49,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421069242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:49,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:49,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:49,258 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:49,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:49,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421069253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:49,260 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:49,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:49,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421069253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:49,262 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:49,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T14:16:49,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-12T14:16:49,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:49,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:49,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:49,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:49,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:49,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:49,454 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/297ffb33b2964320b29c315ca171b6de as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/297ffb33b2964320b29c315ca171b6de 2024-11-12T14:16:49,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/d1244190df6840948e3a846e9e7dde13 2024-11-12T14:16:49,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-12T14:16:49,510 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6ea61e7c1d124f78e4a5241afdaab254/C of 6ea61e7c1d124f78e4a5241afdaab254 into 297ffb33b2964320b29c315ca171b6de(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:16:49,510 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:49,510 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., storeName=6ea61e7c1d124f78e4a5241afdaab254/C, priority=13, startTime=1731421008684; duration=0sec 2024-11-12T14:16:49,511 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:16:49,512 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ea61e7c1d124f78e4a5241afdaab254:C 2024-11-12T14:16:49,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:49,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:49,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:49,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:49,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:49,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:49,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/b754eeb71830426fb7f9e21aaa5712cf as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/b754eeb71830426fb7f9e21aaa5712cf 2024-11-12T14:16:49,566 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/b754eeb71830426fb7f9e21aaa5712cf, entries=200, sequenceid=211, filesize=14.2 K 2024-11-12T14:16:49,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/9feee99b998747f5a286d5670f1323a2 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/9feee99b998747f5a286d5670f1323a2 2024-11-12T14:16:49,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:49,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421069563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:49,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:49,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:49,590 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:49,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:49,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421069570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:49,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:49,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:49,594 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:49,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:49,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421069572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:49,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:49,595 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:49,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:49,595 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:49,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421069576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:49,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:49,616 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/9feee99b998747f5a286d5670f1323a2, entries=150, sequenceid=211, filesize=11.9 K 2024-11-12T14:16:49,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/d1244190df6840948e3a846e9e7dde13 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/d1244190df6840948e3a846e9e7dde13 2024-11-12T14:16:49,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/d1244190df6840948e3a846e9e7dde13, entries=150, sequenceid=211, filesize=11.9 K 2024-11-12T14:16:49,656 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~136.42 KB/139690, heapSize ~401.52 KB/411152, currentSize=46.96 KB/48090 for 6ea61e7c1d124f78e4a5241afdaab254 in 954ms, sequenceid=211, compaction requested=false 2024-11-12T14:16:49,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:49,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-12T14:16:49,670 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:49,671 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 6ea61e7c1d124f78e4a5241afdaab254 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T14:16:49,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=A 2024-11-12T14:16:49,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:49,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=B 2024-11-12T14:16:49,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:49,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=C 2024-11-12T14:16:49,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:49,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/6ceaf1c6215d4f8185505e5bcee4529f is 50, key is test_row_0/A:col10/1731421008786/Put/seqid=0 2024-11-12T14:16:49,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741882_1058 (size=12151) 2024-11-12T14:16:49,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:49,755 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:49,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T14:16:50,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731421010019, max timestamp=1731421010113Num uniques 859; 2024-11-12T14:16:50,124 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T14:16:50,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731421010019, max timestamp=1731421010113Num uniques 859; 2024-11-12T14:16:50,126 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731421010019, max timestamp=1731421010113Num uniques 859; 2024-11-12T14:16:50,127 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T14:16:50,128 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T14:16:50,162 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/6ceaf1c6215d4f8185505e5bcee4529f 2024-11-12T14:16:50,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/f0dcfa9d79ac405ca4d52a0d444e7fa6 is 50, key is test_row_0/B:col10/1731421008786/Put/seqid=0 2024-11-12T14:16:50,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:50,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421070216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:50,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,240 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:50,251 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:50,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421070233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:50,252 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:50,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421070234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:50,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,252 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:50,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:50,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421070235, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:50,262 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421070238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,263 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:50,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,264 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:50,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741883_1059 (size=12151) 2024-11-12T14:16:50,361 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:50,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421070350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,363 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:50,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:50,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421070360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:50,367 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,368 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:50,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:50,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421070372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:50,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,399 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:50,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:50,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421070372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:50,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:50,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:50,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421070370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:50,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,406 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:50,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:50,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421070570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:50,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,580 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,581 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:50,585 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:50,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421070581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:50,588 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,589 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:50,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:50,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421070611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:50,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,617 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:50,617 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:50,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421070612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:50,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:50,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421070620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:50,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,621 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:50,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:50,702 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/f0dcfa9d79ac405ca4d52a0d444e7fa6 2024-11-12T14:16:50,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/0a4acbca7d044fde99b4ce4deee0500f is 50, key is test_row_0/C:col10/1731421008786/Put/seqid=0 2024-11-12T14:16:50,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741884_1060 (size=12151) 2024-11-12T14:16:50,824 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/0a4acbca7d044fde99b4ce4deee0500f 2024-11-12T14:16:50,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T14:16:50,875 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/6ceaf1c6215d4f8185505e5bcee4529f as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/6ceaf1c6215d4f8185505e5bcee4529f 2024-11-12T14:16:50,896 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:50,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421070892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,897 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:50,909 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/6ceaf1c6215d4f8185505e5bcee4529f, entries=150, sequenceid=224, filesize=11.9 K 2024-11-12T14:16:50,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/f0dcfa9d79ac405ca4d52a0d444e7fa6 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/f0dcfa9d79ac405ca4d52a0d444e7fa6 2024-11-12T14:16:50,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:50,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421070902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:50,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,928 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,929 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:50,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:50,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421070920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:50,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,940 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,941 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:50,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:50,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421070930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:50,942 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/f0dcfa9d79ac405ca4d52a0d444e7fa6, entries=150, sequenceid=224, filesize=11.9 K 2024-11-12T14:16:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,942 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,942 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:50,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421070930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:50,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,943 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:50,944 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:50,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/0a4acbca7d044fde99b4ce4deee0500f as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/0a4acbca7d044fde99b4ce4deee0500f 2024-11-12T14:16:50,966 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/0a4acbca7d044fde99b4ce4deee0500f, entries=150, sequenceid=224, filesize=11.9 K 2024-11-12T14:16:50,972 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 6ea61e7c1d124f78e4a5241afdaab254 in 1301ms, sequenceid=224, compaction requested=true 2024-11-12T14:16:50,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:50,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:50,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-12T14:16:50,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-12T14:16:50,984 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-12T14:16:50,984 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2700 sec 2024-11-12T14:16:51,001 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.2960 sec 2024-11-12T14:16:51,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:51,416 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ea61e7c1d124f78e4a5241afdaab254 3/3 column families, dataSize=140.89 KB heapSize=400.22 KB 2024-11-12T14:16:51,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=A 2024-11-12T14:16:51,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:16:51,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=B 2024-11-12T14:16:51,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:16:51,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=C 2024-11-12T14:16:51,417 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:16:51,444 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/2fa4df8bdce8445b9d2e135f05316d9a is 50, key is test_row_0/A:col10/1731421010215/Put/seqid=0 2024-11-12T14:16:51,465 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421071450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:51,465 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:51,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421071450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:51,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:51,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:51,466 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:51,466 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:51,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421071455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:51,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:51,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:51,467 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:51,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:51,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:51,470 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:51,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421071466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:51,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:51,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:51,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:51,479 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:51,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421071469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:51,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:51,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:51,480 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:51,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741885_1061 (size=14641) 2024-11-12T14:16:51,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:51,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421071570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:51,577 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:51,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:51,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421071570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:51,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:51,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:51,579 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:51,591 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:51,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421071590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:51,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:51,593 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:51,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:51,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421071590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:51,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:51,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:51,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:51,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:51,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:51,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421071790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:51,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:51,797 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:51,798 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:51,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:51,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421071790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:51,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:51,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:51,799 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:51,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:51,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421071801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:51,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:51,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:51,804 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:51,804 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:51,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421071801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:51,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:51,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:51,806 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:51,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/2fa4df8bdce8445b9d2e135f05316d9a 2024-11-12T14:16:51,948 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/386ab9e3fc614d43829a1fdbd3caae75 is 50, key is test_row_0/B:col10/1731421010215/Put/seqid=0 2024-11-12T14:16:51,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741886_1062 (size=12251) 2024-11-12T14:16:52,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:52,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421072110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:52,117 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:52,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:52,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421072110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:52,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:52,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:52,119 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:52,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:52,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421072112, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:52,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:52,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:52,121 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:52,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:52,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421072122, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:52,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:52,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:52,130 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:52,399 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/386ab9e3fc614d43829a1fdbd3caae75 2024-11-12T14:16:52,410 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/72f3ec3f489e413c8c3209dfa3ab1ae9 is 50, key is test_row_0/C:col10/1731421010215/Put/seqid=0 2024-11-12T14:16:52,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741887_1063 (size=12251) 2024-11-12T14:16:52,420 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=261 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/72f3ec3f489e413c8c3209dfa3ab1ae9 2024-11-12T14:16:52,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/2fa4df8bdce8445b9d2e135f05316d9a as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/2fa4df8bdce8445b9d2e135f05316d9a 2024-11-12T14:16:52,443 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/2fa4df8bdce8445b9d2e135f05316d9a, entries=200, sequenceid=261, filesize=14.3 K 2024-11-12T14:16:52,444 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/386ab9e3fc614d43829a1fdbd3caae75 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/386ab9e3fc614d43829a1fdbd3caae75 2024-11-12T14:16:52,451 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/386ab9e3fc614d43829a1fdbd3caae75, entries=150, sequenceid=261, filesize=12.0 K 2024-11-12T14:16:52,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/72f3ec3f489e413c8c3209dfa3ab1ae9 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/72f3ec3f489e413c8c3209dfa3ab1ae9 2024-11-12T14:16:52,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/72f3ec3f489e413c8c3209dfa3ab1ae9, entries=150, sequenceid=261, filesize=12.0 K 2024-11-12T14:16:52,477 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~400.17 KB/409776, currentSize=46.96 KB/48090 for 6ea61e7c1d124f78e4a5241afdaab254 in 1061ms, sequenceid=261, compaction requested=true 2024-11-12T14:16:52,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:52,477 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:16:52,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ea61e7c1d124f78e4a5241afdaab254:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:16:52,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:16:52,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ea61e7c1d124f78e4a5241afdaab254:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:16:52,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:16:52,478 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:16:52,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ea61e7c1d124f78e4a5241afdaab254:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:16:52,478 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:16:52,480 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53928 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:16:52,480 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 6ea61e7c1d124f78e4a5241afdaab254/A is initiating minor compaction (all files) 2024-11-12T14:16:52,481 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ea61e7c1d124f78e4a5241afdaab254/A in TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:52,481 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/ccfc6d3beaf144679f45b7eaa984aded, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/b754eeb71830426fb7f9e21aaa5712cf, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/6ceaf1c6215d4f8185505e5bcee4529f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/2fa4df8bdce8445b9d2e135f05316d9a] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp, totalSize=52.7 K 2024-11-12T14:16:52,482 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49148 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:16:52,483 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 6ea61e7c1d124f78e4a5241afdaab254/B is initiating minor compaction (all files) 2024-11-12T14:16:52,483 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ea61e7c1d124f78e4a5241afdaab254/B in TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:52,483 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/435d7cfdb6474d18859d44263c84fac5, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/9feee99b998747f5a286d5670f1323a2, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/f0dcfa9d79ac405ca4d52a0d444e7fa6, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/386ab9e3fc614d43829a1fdbd3caae75] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp, totalSize=48.0 K 2024-11-12T14:16:52,483 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting ccfc6d3beaf144679f45b7eaa984aded, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731421008317 2024-11-12T14:16:52,485 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 435d7cfdb6474d18859d44263c84fac5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731421008317 2024-11-12T14:16:52,485 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b754eeb71830426fb7f9e21aaa5712cf, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731421008447 2024-11-12T14:16:52,485 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 9feee99b998747f5a286d5670f1323a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731421008447 2024-11-12T14:16:52,486 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ceaf1c6215d4f8185505e5bcee4529f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731421008786 2024-11-12T14:16:52,486 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting f0dcfa9d79ac405ca4d52a0d444e7fa6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731421008786 2024-11-12T14:16:52,487 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2fa4df8bdce8445b9d2e135f05316d9a, keycount=200, bloomtype=ROW, size=14.3 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731421010213 2024-11-12T14:16:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:52,488 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ea61e7c1d124f78e4a5241afdaab254 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T14:16:52,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=A 2024-11-12T14:16:52,489 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 386ab9e3fc614d43829a1fdbd3caae75, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731421010215 2024-11-12T14:16:52,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:52,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=B 2024-11-12T14:16:52,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:52,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=C 2024-11-12T14:16:52,489 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:52,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/0986dd238b1f4feb8a6074d6759709bb is 50, key is test_row_0/A:col10/1731421011445/Put/seqid=0 2024-11-12T14:16:52,522 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ea61e7c1d124f78e4a5241afdaab254#A#compaction#52 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:16:52,523 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/8ec7d3b356ac40049193d1a6a1c21f9a is 50, key is test_row_0/A:col10/1731421010215/Put/seqid=0 2024-11-12T14:16:52,531 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ea61e7c1d124f78e4a5241afdaab254#B#compaction#53 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:16:52,532 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/2d97ed30968248179aa09d120c3d96bf is 50, key is test_row_0/B:col10/1731421010215/Put/seqid=0 2024-11-12T14:16:52,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741889_1065 (size=12831) 2024-11-12T14:16:52,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741888_1064 (size=14741) 2024-11-12T14:16:52,565 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/0986dd238b1f4feb8a6074d6759709bb 2024-11-12T14:16:52,579 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/8ec7d3b356ac40049193d1a6a1c21f9a as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/8ec7d3b356ac40049193d1a6a1c21f9a 2024-11-12T14:16:52,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741890_1066 (size=12831) 2024-11-12T14:16:52,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/f97cfc95673349e78b1a5dba9476e039 is 50, key is test_row_0/B:col10/1731421011445/Put/seqid=0 2024-11-12T14:16:52,597 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/2d97ed30968248179aa09d120c3d96bf as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/2d97ed30968248179aa09d120c3d96bf 2024-11-12T14:16:52,599 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ea61e7c1d124f78e4a5241afdaab254/A of 6ea61e7c1d124f78e4a5241afdaab254 into 8ec7d3b356ac40049193d1a6a1c21f9a(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:16:52,599 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:52,600 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., storeName=6ea61e7c1d124f78e4a5241afdaab254/A, priority=12, startTime=1731421012477; duration=0sec 2024-11-12T14:16:52,600 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:16:52,600 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ea61e7c1d124f78e4a5241afdaab254:A 2024-11-12T14:16:52,600 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:16:52,604 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49148 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:16:52,604 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 6ea61e7c1d124f78e4a5241afdaab254/C is initiating minor compaction (all files) 2024-11-12T14:16:52,604 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ea61e7c1d124f78e4a5241afdaab254/C in TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:52,604 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/297ffb33b2964320b29c315ca171b6de, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/d1244190df6840948e3a846e9e7dde13, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/0a4acbca7d044fde99b4ce4deee0500f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/72f3ec3f489e413c8c3209dfa3ab1ae9] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp, totalSize=48.0 K 2024-11-12T14:16:52,605 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 297ffb33b2964320b29c315ca171b6de, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731421008317 2024-11-12T14:16:52,605 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting d1244190df6840948e3a846e9e7dde13, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1731421008447 2024-11-12T14:16:52,606 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a4acbca7d044fde99b4ce4deee0500f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731421008786 2024-11-12T14:16:52,606 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72f3ec3f489e413c8c3209dfa3ab1ae9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731421010215 2024-11-12T14:16:52,613 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ea61e7c1d124f78e4a5241afdaab254/B of 6ea61e7c1d124f78e4a5241afdaab254 into 2d97ed30968248179aa09d120c3d96bf(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:16:52,613 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:52,613 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., storeName=6ea61e7c1d124f78e4a5241afdaab254/B, priority=12, startTime=1731421012478; duration=0sec 2024-11-12T14:16:52,613 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:16:52,613 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ea61e7c1d124f78e4a5241afdaab254:B 2024-11-12T14:16:52,622 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741891_1067 (size=12301) 2024-11-12T14:16:52,623 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/f97cfc95673349e78b1a5dba9476e039 2024-11-12T14:16:52,629 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ea61e7c1d124f78e4a5241afdaab254#C#compaction#55 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:16:52,630 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/6228b841b5324d24b099bbdcb9678fcb is 50, key is test_row_0/C:col10/1731421010215/Put/seqid=0 2024-11-12T14:16:52,648 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/67bf68ab221340f8a04d5b8acb428a9f is 50, key is test_row_0/C:col10/1731421011445/Put/seqid=0 2024-11-12T14:16:52,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741892_1068 (size=12831) 2024-11-12T14:16:52,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741893_1069 (size=12301) 2024-11-12T14:16:52,658 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/6228b841b5324d24b099bbdcb9678fcb as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/6228b841b5324d24b099bbdcb9678fcb 2024-11-12T14:16:52,691 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ea61e7c1d124f78e4a5241afdaab254/C of 6ea61e7c1d124f78e4a5241afdaab254 into 6228b841b5324d24b099bbdcb9678fcb(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:16:52,691 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:52,691 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., storeName=6ea61e7c1d124f78e4a5241afdaab254/C, priority=12, startTime=1731421012478; duration=0sec 2024-11-12T14:16:52,691 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:16:52,691 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ea61e7c1d124f78e4a5241afdaab254:C 2024-11-12T14:16:52,695 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:52,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421072642, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:52,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731421012605, max timestamp=1731421012692Num uniques 859; 2024-11-12T14:16:52,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731421012605, max timestamp=1731421012692Num uniques 859; 2024-11-12T14:16:52,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T14:16:52,697 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T14:16:52,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731421012605, max timestamp=1731421012692Num uniques 859; 2024-11-12T14:16:52,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T14:16:52,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:52,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:52,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:52,763 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:52,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421072743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:52,765 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:52,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421072743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:52,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:52,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:52,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:52,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:52,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421072764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:52,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:52,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:52,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:52,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:52,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421072789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:52,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:52,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:52,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:52,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:52,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421072809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:52,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:52,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:52,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:52,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:52,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421072869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:52,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:52,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421072870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:52,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:52,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:52,879 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:52,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-12T14:16:52,879 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:16:52,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:52,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:52,880 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:16:52,880 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:52,884 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:52,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-12T14:16:52,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421072879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:52,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:52,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:52,886 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:52,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T14:16:52,886 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:16:52,887 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:16:52,888 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:16:52,903 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:52,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421072900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:52,904 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:52,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T14:16:53,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:53,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421073021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:53,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:53,025 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:53,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T14:16:53,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:53,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:53,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:53,049 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:53,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:53,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:53,065 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/67bf68ab221340f8a04d5b8acb428a9f 2024-11-12T14:16:53,084 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/0986dd238b1f4feb8a6074d6759709bb as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/0986dd238b1f4feb8a6074d6759709bb 2024-11-12T14:16:53,095 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/0986dd238b1f4feb8a6074d6759709bb, entries=200, sequenceid=272, filesize=14.4 K 2024-11-12T14:16:53,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:53,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421073090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:53,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:53,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:53,097 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:53,098 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:53,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421073091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:53,099 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:53,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:53,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421073092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:53,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:53,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:53,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:53,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/f97cfc95673349e78b1a5dba9476e039 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/f97cfc95673349e78b1a5dba9476e039 2024-11-12T14:16:53,116 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:53,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421073109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:53,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:53,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:53,117 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:53,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/f97cfc95673349e78b1a5dba9476e039, entries=150, sequenceid=272, filesize=12.0 K 2024-11-12T14:16:53,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/67bf68ab221340f8a04d5b8acb428a9f as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/67bf68ab221340f8a04d5b8acb428a9f 2024-11-12T14:16:53,157 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/67bf68ab221340f8a04d5b8acb428a9f, entries=150, sequenceid=272, filesize=12.0 K 2024-11-12T14:16:53,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=134.18 KB/137400 for 6ea61e7c1d124f78e4a5241afdaab254 in 673ms, sequenceid=272, compaction requested=false 2024-11-12T14:16:53,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:53,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-12T14:16:53,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:53,210 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 6ea61e7c1d124f78e4a5241afdaab254 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-12T14:16:53,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=A 2024-11-12T14:16:53,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:16:53,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=B 2024-11-12T14:16:53,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:16:53,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=C 2024-11-12T14:16:53,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:16:53,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T14:16:53,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/0de10473ad744c689d11c8faad9e8440 is 50, key is test_row_0/A:col10/1731421012758/Put/seqid=0 2024-11-12T14:16:53,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741894_1070 (size=12301) 2024-11-12T14:16:53,289 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/0de10473ad744c689d11c8faad9e8440 2024-11-12T14:16:53,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/d91ced07fee340a8a5e5cd6ff275a024 is 50, key is test_row_0/B:col10/1731421012758/Put/seqid=0 2024-11-12T14:16:53,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741895_1071 (size=12301) 2024-11-12T14:16:53,348 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:53,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:53,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:53,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421073413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:53,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:53,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:53,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:53,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:53,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421073416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:53,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:53,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:53,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:53,431 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:53,431 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:53,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421073417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:53,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421073418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:53,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:53,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:53,431 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:53,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:53,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:53,432 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:53,432 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:53,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421073419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:53,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:53,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:53,432 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:53,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T14:16:53,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:53,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421073542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:53,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:53,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:53,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:53,552 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:53,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421073542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:53,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:53,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:53,553 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:53,741 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/d91ced07fee340a8a5e5cd6ff275a024 2024-11-12T14:16:53,766 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:53,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421073760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:53,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:53,767 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:53,768 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:53,768 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:53,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421073760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:53,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:53,768 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:53,769 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:53,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/7f0d78109eac4c32a9bd155cd077fc22 is 50, key is test_row_0/C:col10/1731421012758/Put/seqid=0 2024-11-12T14:16:53,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741896_1072 (size=12301) 2024-11-12T14:16:53,805 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/7f0d78109eac4c32a9bd155cd077fc22 2024-11-12T14:16:53,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/0de10473ad744c689d11c8faad9e8440 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/0de10473ad744c689d11c8faad9e8440 2024-11-12T14:16:53,860 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/0de10473ad744c689d11c8faad9e8440, entries=150, sequenceid=310, filesize=12.0 K 2024-11-12T14:16:53,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/d91ced07fee340a8a5e5cd6ff275a024 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/d91ced07fee340a8a5e5cd6ff275a024 2024-11-12T14:16:53,882 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/d91ced07fee340a8a5e5cd6ff275a024, entries=150, sequenceid=310, filesize=12.0 K 2024-11-12T14:16:53,883 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/7f0d78109eac4c32a9bd155cd077fc22 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/7f0d78109eac4c32a9bd155cd077fc22 2024-11-12T14:16:53,900 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/7f0d78109eac4c32a9bd155cd077fc22, entries=150, sequenceid=310, filesize=12.0 K 2024-11-12T14:16:53,902 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for 6ea61e7c1d124f78e4a5241afdaab254 in 692ms, sequenceid=310, compaction requested=true 2024-11-12T14:16:53,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:53,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:53,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-12T14:16:53,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-12T14:16:53,907 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-12T14:16:53,907 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0170 sec 2024-11-12T14:16:53,914 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.0280 sec 2024-11-12T14:16:53,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:53,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ea61e7c1d124f78e4a5241afdaab254 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T14:16:53,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=A 2024-11-12T14:16:53,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:53,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=B 2024-11-12T14:16:53,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:53,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=C 2024-11-12T14:16:53,959 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:53,970 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/cd12081b257d405cb9251aac257d3980 is 50, key is test_row_0/A:col10/1731421013354/Put/seqid=0 2024-11-12T14:16:53,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741897_1073 (size=12301) 2024-11-12T14:16:53,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/cd12081b257d405cb9251aac257d3980 2024-11-12T14:16:54,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/48c163760f3a4366aeac534fc06c89a3 is 50, key is test_row_0/B:col10/1731421013354/Put/seqid=0 2024-11-12T14:16:54,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741898_1074 (size=12301) 2024-11-12T14:16:54,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/48c163760f3a4366aeac534fc06c89a3 2024-11-12T14:16:54,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-12T14:16:54,052 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:16:54,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:54,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421074041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:54,053 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:16:54,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:54,053 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:54,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421074043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-12T14:16:54,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:54,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,057 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421074044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:54,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T14:16:54,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,058 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:54,058 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:16:54,059 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:16:54,059 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:16:54,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/5cba51deba6640e0bfe664fe29f2ae84 is 50, key is test_row_0/C:col10/1731421013354/Put/seqid=0 2024-11-12T14:16:54,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741899_1075 (size=12301) 2024-11-12T14:16:54,080 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/5cba51deba6640e0bfe664fe29f2ae84 2024-11-12T14:16:54,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:54,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421074080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:54,083 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:54,084 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:54,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421074080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:54,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,084 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:54,093 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/cd12081b257d405cb9251aac257d3980 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/cd12081b257d405cb9251aac257d3980 2024-11-12T14:16:54,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/cd12081b257d405cb9251aac257d3980, entries=150, sequenceid=324, filesize=12.0 K 2024-11-12T14:16:54,113 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/48c163760f3a4366aeac534fc06c89a3 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/48c163760f3a4366aeac534fc06c89a3 2024-11-12T14:16:54,122 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/48c163760f3a4366aeac534fc06c89a3, entries=150, sequenceid=324, filesize=12.0 K 2024-11-12T14:16:54,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/5cba51deba6640e0bfe664fe29f2ae84 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/5cba51deba6640e0bfe664fe29f2ae84 2024-11-12T14:16:54,149 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/5cba51deba6640e0bfe664fe29f2ae84, entries=150, sequenceid=324, filesize=12.0 K 2024-11-12T14:16:54,151 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 6ea61e7c1d124f78e4a5241afdaab254 in 194ms, sequenceid=324, compaction requested=true 2024-11-12T14:16:54,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:54,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ea61e7c1d124f78e4a5241afdaab254:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:16:54,151 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:16:54,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:16:54,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ea61e7c1d124f78e4a5241afdaab254:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:16:54,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:16:54,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ea61e7c1d124f78e4a5241afdaab254:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:16:54,151 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:16:54,151 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:16:54,153 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49734 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:16:54,153 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 6ea61e7c1d124f78e4a5241afdaab254/B is initiating minor compaction (all files) 2024-11-12T14:16:54,154 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ea61e7c1d124f78e4a5241afdaab254/B in TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:54,154 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/2d97ed30968248179aa09d120c3d96bf, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/f97cfc95673349e78b1a5dba9476e039, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/d91ced07fee340a8a5e5cd6ff275a024, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/48c163760f3a4366aeac534fc06c89a3] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp, totalSize=48.6 K 2024-11-12T14:16:54,159 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d97ed30968248179aa09d120c3d96bf, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731421010215 2024-11-12T14:16:54,159 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52174 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:16:54,159 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting f97cfc95673349e78b1a5dba9476e039, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731421011445 2024-11-12T14:16:54,159 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 6ea61e7c1d124f78e4a5241afdaab254/A is initiating minor compaction (all files) 2024-11-12T14:16:54,159 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ea61e7c1d124f78e4a5241afdaab254/A in TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:54,160 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/8ec7d3b356ac40049193d1a6a1c21f9a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/0986dd238b1f4feb8a6074d6759709bb, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/0de10473ad744c689d11c8faad9e8440, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/cd12081b257d405cb9251aac257d3980] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp, totalSize=51.0 K 2024-11-12T14:16:54,160 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting d91ced07fee340a8a5e5cd6ff275a024, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731421012735 2024-11-12T14:16:54,160 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ec7d3b356ac40049193d1a6a1c21f9a, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731421010215 2024-11-12T14:16:54,160 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 48c163760f3a4366aeac534fc06c89a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731421013354 2024-11-12T14:16:54,160 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0986dd238b1f4feb8a6074d6759709bb, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731421011445 2024-11-12T14:16:54,161 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0de10473ad744c689d11c8faad9e8440, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731421012735 2024-11-12T14:16:54,161 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd12081b257d405cb9251aac257d3980, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731421013354 2024-11-12T14:16:54,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:54,167 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ea61e7c1d124f78e4a5241afdaab254 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T14:16:54,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=A 2024-11-12T14:16:54,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:54,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=B 2024-11-12T14:16:54,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:54,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=C 2024-11-12T14:16:54,168 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:54,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T14:16:54,182 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/222bba6d92604c00ba1a54e8455154f7 is 50, key is test_row_0/A:col10/1731421014001/Put/seqid=0 2024-11-12T14:16:54,182 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ea61e7c1d124f78e4a5241afdaab254#B#compaction#63 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:16:54,183 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/33944f5e65eb4d15960ef06d0c847300 is 50, key is test_row_0/B:col10/1731421013354/Put/seqid=0 2024-11-12T14:16:54,184 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ea61e7c1d124f78e4a5241afdaab254#A#compaction#65 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:16:54,184 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/952ac7d7622743bfa96eecbfc30debce is 50, key is test_row_0/A:col10/1731421013354/Put/seqid=0 2024-11-12T14:16:54,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:54,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421074190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:54,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,194 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:54,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421074192, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:54,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:54,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421074194, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:54,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,201 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:54,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T14:16:54,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:54,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:54,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:54,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:54,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:54,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:54,223 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741900_1076 (size=12301) 2024-11-12T14:16:54,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741901_1077 (size=13017) 2024-11-12T14:16:54,279 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/952ac7d7622743bfa96eecbfc30debce as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/952ac7d7622743bfa96eecbfc30debce 2024-11-12T14:16:54,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741902_1078 (size=13017) 2024-11-12T14:16:54,307 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:54,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421074300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:54,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,313 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:54,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:54,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421074311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:54,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:54,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,323 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:54,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421074312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:54,324 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/33944f5e65eb4d15960ef06d0c847300 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/33944f5e65eb4d15960ef06d0c847300 2024-11-12T14:16:54,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,324 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:54,336 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ea61e7c1d124f78e4a5241afdaab254/B of 6ea61e7c1d124f78e4a5241afdaab254 into 33944f5e65eb4d15960ef06d0c847300(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:16:54,336 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:54,336 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., storeName=6ea61e7c1d124f78e4a5241afdaab254/B, priority=12, startTime=1731421014151; duration=0sec 2024-11-12T14:16:54,336 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:16:54,336 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ea61e7c1d124f78e4a5241afdaab254:B 2024-11-12T14:16:54,337 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:16:54,338 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ea61e7c1d124f78e4a5241afdaab254/A of 6ea61e7c1d124f78e4a5241afdaab254 into 952ac7d7622743bfa96eecbfc30debce(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:16:54,338 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:54,338 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., storeName=6ea61e7c1d124f78e4a5241afdaab254/A, priority=12, startTime=1731421014151; duration=0sec 2024-11-12T14:16:54,338 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:16:54,338 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ea61e7c1d124f78e4a5241afdaab254:A 2024-11-12T14:16:54,345 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49734 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:16:54,345 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 6ea61e7c1d124f78e4a5241afdaab254/C is initiating minor compaction (all files) 2024-11-12T14:16:54,345 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ea61e7c1d124f78e4a5241afdaab254/C in TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:54,345 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/6228b841b5324d24b099bbdcb9678fcb, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/67bf68ab221340f8a04d5b8acb428a9f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/7f0d78109eac4c32a9bd155cd077fc22, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/5cba51deba6640e0bfe664fe29f2ae84] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp, totalSize=48.6 K 2024-11-12T14:16:54,346 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 6228b841b5324d24b099bbdcb9678fcb, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=261, earliestPutTs=1731421010215 2024-11-12T14:16:54,347 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 67bf68ab221340f8a04d5b8acb428a9f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731421011445 2024-11-12T14:16:54,348 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f0d78109eac4c32a9bd155cd077fc22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731421012735 2024-11-12T14:16:54,348 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cba51deba6640e0bfe664fe29f2ae84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731421013354 2024-11-12T14:16:54,367 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T14:16:54,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:54,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:54,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:54,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:54,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:54,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:54,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T14:16:54,383 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ea61e7c1d124f78e4a5241afdaab254#C#compaction#66 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:16:54,384 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/a01461c9f6724f9ca5fc0492c5329aff is 50, key is test_row_0/C:col10/1731421013354/Put/seqid=0 2024-11-12T14:16:54,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741903_1079 (size=13017) 2024-11-12T14:16:54,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:54,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421074519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:54,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:54,522 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T14:16:54,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:54,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:54,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:54,523 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:54,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:54,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:54,535 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:54,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421074530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:54,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,538 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:54,539 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:54,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421074530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:54,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,541 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:54,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421074589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:54,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421074591, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:54,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,596 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:54,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,600 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:54,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/222bba6d92604c00ba1a54e8455154f7 2024-11-12T14:16:54,640 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/4b6a3530f9904a09b66c2e17ff31ae0c is 50, key is test_row_0/B:col10/1731421014001/Put/seqid=0 2024-11-12T14:16:54,676 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T14:16:54,676 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:54,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:54,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:54,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:54,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:54,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741904_1080 (size=12301) 2024-11-12T14:16:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T14:16:54,830 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T14:16:54,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:54,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:54,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:54,831 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:54,831 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:54,835 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/a01461c9f6724f9ca5fc0492c5329aff as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/a01461c9f6724f9ca5fc0492c5329aff 2024-11-12T14:16:54,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421074829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:54,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:54,846 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ea61e7c1d124f78e4a5241afdaab254/C of 6ea61e7c1d124f78e4a5241afdaab254 into a01461c9f6724f9ca5fc0492c5329aff(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:16:54,846 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:54,846 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., storeName=6ea61e7c1d124f78e4a5241afdaab254/C, priority=12, startTime=1731421014151; duration=0sec 2024-11-12T14:16:54,846 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:16:54,846 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ea61e7c1d124f78e4a5241afdaab254:C 2024-11-12T14:16:54,856 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:54,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421074850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:54,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:54,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,857 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:54,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421074850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:54,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:54,859 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:54,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T14:16:54,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:54,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:54,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:54,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:54,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:54,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:55,098 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/4b6a3530f9904a09b66c2e17ff31ae0c 2024-11-12T14:16:55,145 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/9f0279c92ff543ca9e77b880208d6ca2 is 50, key is test_row_0/C:col10/1731421014001/Put/seqid=0 2024-11-12T14:16:55,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T14:16:55,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:55,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:55,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:55,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:55,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:55,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:16:55,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741905_1081 (size=12301) 2024-11-12T14:16:55,176 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/9f0279c92ff543ca9e77b880208d6ca2 2024-11-12T14:16:55,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/222bba6d92604c00ba1a54e8455154f7 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/222bba6d92604c00ba1a54e8455154f7 2024-11-12T14:16:55,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T14:16:55,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/222bba6d92604c00ba1a54e8455154f7, entries=150, sequenceid=344, filesize=12.0 K 2024-11-12T14:16:55,202 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/4b6a3530f9904a09b66c2e17ff31ae0c as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/4b6a3530f9904a09b66c2e17ff31ae0c 2024-11-12T14:16:55,223 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/4b6a3530f9904a09b66c2e17ff31ae0c, entries=150, sequenceid=344, filesize=12.0 K 2024-11-12T14:16:55,225 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/9f0279c92ff543ca9e77b880208d6ca2 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/9f0279c92ff543ca9e77b880208d6ca2 2024-11-12T14:16:55,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/9f0279c92ff543ca9e77b880208d6ca2, entries=150, sequenceid=344, filesize=12.0 K 2024-11-12T14:16:55,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 6ea61e7c1d124f78e4a5241afdaab254 in 1074ms, sequenceid=344, compaction requested=false 2024-11-12T14:16:55,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:55,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-12T14:16:55,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:55,305 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 6ea61e7c1d124f78e4a5241afdaab254 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T14:16:55,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=A 2024-11-12T14:16:55,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:55,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=B 2024-11-12T14:16:55,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:55,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=C 2024-11-12T14:16:55,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:55,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/8ebf6365bb504a40ab199728e5b16a6e is 50, key is test_row_0/A:col10/1731421014185/Put/seqid=0 2024-11-12T14:16:55,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:55,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. as already flushing 2024-11-12T14:16:55,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741906_1082 (size=12301) 2024-11-12T14:16:55,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:55,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421075464, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:55,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:55,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:55,473 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:55,478 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:55,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421075468, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:55,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:55,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:55,479 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:55,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:55,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421075473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:55,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:55,486 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:55,487 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:55,583 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:55,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421075580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:55,586 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:55,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421075589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:55,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:55,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:55,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421075589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:55,598 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:55,613 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:55,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421075609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:55,614 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_1' in 6ea61e7c1d124f78e4a5241afdaab254 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:16:55,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:55,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:55,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:55,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:55,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421075619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:55,624 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_1' in 6ea61e7c1d124f78e4a5241afdaab254 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:16:55,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:55,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:55,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:55,771 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/8ebf6365bb504a40ab199728e5b16a6e 2024-11-12T14:16:55,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:55,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421075790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:55,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:55,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:55,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:55,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/a1068de709534f249eaa55254d0a2d1c is 50, key is test_row_0/B:col10/1731421014185/Put/seqid=0 2024-11-12T14:16:55,817 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:55,817 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421075810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:55,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421075810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:55,817 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:55,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:55,818 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:55,819 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:55,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:55,820 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:55,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741907_1083 (size=12301) 2024-11-12T14:16:55,842 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/a1068de709534f249eaa55254d0a2d1c 2024-11-12T14:16:55,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/047fa94674c6411d8a914f84d13ebe19 is 50, key is test_row_0/C:col10/1731421014185/Put/seqid=0 2024-11-12T14:16:55,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741908_1084 (size=12301) 2024-11-12T14:16:56,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:56,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421076100, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:56,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:56,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:56,114 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:56,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:56,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421076129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:56,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:56,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:56,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:56,140 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:56,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421076132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:56,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:56,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T14:16:56,293 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/047fa94674c6411d8a914f84d13ebe19 2024-11-12T14:16:56,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/8ebf6365bb504a40ab199728e5b16a6e as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/8ebf6365bb504a40ab199728e5b16a6e 2024-11-12T14:16:56,365 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/8ebf6365bb504a40ab199728e5b16a6e, entries=150, sequenceid=359, filesize=12.0 K 2024-11-12T14:16:56,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/a1068de709534f249eaa55254d0a2d1c as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/a1068de709534f249eaa55254d0a2d1c 2024-11-12T14:16:56,392 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/a1068de709534f249eaa55254d0a2d1c, entries=150, sequenceid=359, filesize=12.0 K 2024-11-12T14:16:56,399 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/047fa94674c6411d8a914f84d13ebe19 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/047fa94674c6411d8a914f84d13ebe19 2024-11-12T14:16:56,434 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/047fa94674c6411d8a914f84d13ebe19, entries=150, sequenceid=359, filesize=12.0 K 2024-11-12T14:16:56,437 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 6ea61e7c1d124f78e4a5241afdaab254 in 1132ms, sequenceid=359, compaction requested=true 2024-11-12T14:16:56,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:56,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:56,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-12T14:16:56,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-12T14:16:56,442 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-12T14:16:56,442 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3800 sec 2024-11-12T14:16:56,445 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 2.3900 sec 2024-11-12T14:16:56,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731421015418, max timestamp=1731421016624Num uniques 859; 2024-11-12T14:16:56,628 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T14:16:56,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:56,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ea61e7c1d124f78e4a5241afdaab254 3/3 column families, dataSize=89.45 KB heapSize=262.11 KB 2024-11-12T14:16:56,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=A 2024-11-12T14:16:56,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:16:56,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=B 2024-11-12T14:16:56,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:16:56,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=C 2024-11-12T14:16:56,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:16:56,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/7bfd4295ce48402792e82e65108729a8 is 50, key is test_row_0/A:col10/1731421015469/Put/seqid=0 2024-11-12T14:16:56,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741909_1085 (size=14741) 2024-11-12T14:16:56,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:56,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421076704, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:56,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:56,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:56,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421076706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:56,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:56,718 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:56,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:56,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:56,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:56,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:56,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421076708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:56,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:56,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:56,722 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:56,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:56,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421076830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:56,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:56,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421076830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:56,837 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:56,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:56,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421076831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:56,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:56,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:56,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:57,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:57,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421077039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:57,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:57,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:57,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:57,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421077050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:57,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:57,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:57,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:57,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:57,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421077052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:57,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:57,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:57,054 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:57,103 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/7bfd4295ce48402792e82e65108729a8 2024-11-12T14:16:57,121 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/e7c73010aaa24626861671db2e006de0 is 50, key is test_row_0/B:col10/1731421015469/Put/seqid=0 2024-11-12T14:16:57,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741910_1086 (size=12301) 2024-11-12T14:16:57,210 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/e7c73010aaa24626861671db2e006de0 2024-11-12T14:16:57,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/69c6010737f04c50b8f752eb48bcc8e2 is 50, key is test_row_0/C:col10/1731421015469/Put/seqid=0 2024-11-12T14:16:57,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741911_1087 (size=12301) 2024-11-12T14:16:57,291 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/69c6010737f04c50b8f752eb48bcc8e2 2024-11-12T14:16:57,305 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/7bfd4295ce48402792e82e65108729a8 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/7bfd4295ce48402792e82e65108729a8 2024-11-12T14:16:57,314 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/7bfd4295ce48402792e82e65108729a8, entries=200, sequenceid=380, filesize=14.4 K 2024-11-12T14:16:57,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/e7c73010aaa24626861671db2e006de0 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/e7c73010aaa24626861671db2e006de0 2024-11-12T14:16:57,326 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/e7c73010aaa24626861671db2e006de0, entries=150, sequenceid=380, filesize=12.0 K 2024-11-12T14:16:57,330 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/69c6010737f04c50b8f752eb48bcc8e2 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/69c6010737f04c50b8f752eb48bcc8e2 2024-11-12T14:16:57,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:57,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421077350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:57,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:57,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:57,352 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:57,365 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:57,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421077361, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:57,367 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:57,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:57,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:57,368 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:57,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421077365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:57,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:57,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:57,374 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:57,385 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/69c6010737f04c50b8f752eb48bcc8e2, entries=150, sequenceid=380, filesize=12.0 K 2024-11-12T14:16:57,388 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~89.45 KB/91600, heapSize ~262.06 KB/268352, currentSize=87.22 KB/89310 for 6ea61e7c1d124f78e4a5241afdaab254 in 759ms, sequenceid=380, compaction requested=true 2024-11-12T14:16:57,388 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:57,388 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:16:57,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ea61e7c1d124f78e4a5241afdaab254:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:16:57,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:16:57,390 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:16:57,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ea61e7c1d124f78e4a5241afdaab254:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:16:57,390 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:16:57,392 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:16:57,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ea61e7c1d124f78e4a5241afdaab254:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:16:57,392 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 6ea61e7c1d124f78e4a5241afdaab254/A is initiating minor compaction (all files) 2024-11-12T14:16:57,392 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:16:57,392 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ea61e7c1d124f78e4a5241afdaab254/A in TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:57,392 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/952ac7d7622743bfa96eecbfc30debce, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/222bba6d92604c00ba1a54e8455154f7, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/8ebf6365bb504a40ab199728e5b16a6e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/7bfd4295ce48402792e82e65108729a8] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp, totalSize=51.1 K 2024-11-12T14:16:57,393 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 952ac7d7622743bfa96eecbfc30debce, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731421013354 2024-11-12T14:16:57,393 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 222bba6d92604c00ba1a54e8455154f7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731421014001 2024-11-12T14:16:57,394 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:16:57,394 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 6ea61e7c1d124f78e4a5241afdaab254/B is initiating minor compaction (all files) 2024-11-12T14:16:57,394 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ea61e7c1d124f78e4a5241afdaab254/B in TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:57,394 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/33944f5e65eb4d15960ef06d0c847300, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/4b6a3530f9904a09b66c2e17ff31ae0c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/a1068de709534f249eaa55254d0a2d1c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/e7c73010aaa24626861671db2e006de0] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp, totalSize=48.8 K 2024-11-12T14:16:57,395 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ebf6365bb504a40ab199728e5b16a6e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731421014185 2024-11-12T14:16:57,395 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 33944f5e65eb4d15960ef06d0c847300, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731421013354 2024-11-12T14:16:57,395 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b6a3530f9904a09b66c2e17ff31ae0c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731421014001 2024-11-12T14:16:57,395 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7bfd4295ce48402792e82e65108729a8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731421015418 2024-11-12T14:16:57,396 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting a1068de709534f249eaa55254d0a2d1c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731421014185 2024-11-12T14:16:57,399 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting e7c73010aaa24626861671db2e006de0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731421015460 2024-11-12T14:16:57,436 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ea61e7c1d124f78e4a5241afdaab254#A#compaction#75 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:16:57,438 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/602078873a0149ffbd71f7b2f607270e is 50, key is test_row_0/A:col10/1731421015469/Put/seqid=0 2024-11-12T14:16:57,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741912_1088 (size=13153) 2024-11-12T14:16:57,484 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ea61e7c1d124f78e4a5241afdaab254#B#compaction#76 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:16:57,485 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/75c06f410d6044689e0e3cedb9307d3f is 50, key is test_row_0/B:col10/1731421015469/Put/seqid=0 2024-11-12T14:16:57,507 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/602078873a0149ffbd71f7b2f607270e as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/602078873a0149ffbd71f7b2f607270e 2024-11-12T14:16:57,517 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ea61e7c1d124f78e4a5241afdaab254/A of 6ea61e7c1d124f78e4a5241afdaab254 into 602078873a0149ffbd71f7b2f607270e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:16:57,517 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:57,517 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., storeName=6ea61e7c1d124f78e4a5241afdaab254/A, priority=12, startTime=1731421017388; duration=0sec 2024-11-12T14:16:57,518 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:16:57,518 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ea61e7c1d124f78e4a5241afdaab254:A 2024-11-12T14:16:57,518 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:16:57,520 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49920 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:16:57,521 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 6ea61e7c1d124f78e4a5241afdaab254/C is initiating minor compaction (all files) 2024-11-12T14:16:57,522 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6ea61e7c1d124f78e4a5241afdaab254/C in TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:57,522 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/a01461c9f6724f9ca5fc0492c5329aff, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/9f0279c92ff543ca9e77b880208d6ca2, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/047fa94674c6411d8a914f84d13ebe19, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/69c6010737f04c50b8f752eb48bcc8e2] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp, totalSize=48.8 K 2024-11-12T14:16:57,523 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting a01461c9f6724f9ca5fc0492c5329aff, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731421013354 2024-11-12T14:16:57,524 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9f0279c92ff543ca9e77b880208d6ca2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731421014001 2024-11-12T14:16:57,524 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 047fa94674c6411d8a914f84d13ebe19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731421014185 2024-11-12T14:16:57,524 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69c6010737f04c50b8f752eb48bcc8e2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731421015460 2024-11-12T14:16:57,530 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741913_1089 (size=13153) 2024-11-12T14:16:57,555 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/75c06f410d6044689e0e3cedb9307d3f as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/75c06f410d6044689e0e3cedb9307d3f 2024-11-12T14:16:57,563 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6ea61e7c1d124f78e4a5241afdaab254#C#compaction#77 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:16:57,563 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/6b01852adec94e018eb1d2f9e8217a88 is 50, key is test_row_0/C:col10/1731421015469/Put/seqid=0 2024-11-12T14:16:57,569 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ea61e7c1d124f78e4a5241afdaab254/B of 6ea61e7c1d124f78e4a5241afdaab254 into 75c06f410d6044689e0e3cedb9307d3f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:16:57,569 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:57,570 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., storeName=6ea61e7c1d124f78e4a5241afdaab254/B, priority=12, startTime=1731421017390; duration=0sec 2024-11-12T14:16:57,570 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:16:57,570 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ea61e7c1d124f78e4a5241afdaab254:B 2024-11-12T14:16:57,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741914_1090 (size=13153) 2024-11-12T14:16:57,600 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/6b01852adec94e018eb1d2f9e8217a88 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/6b01852adec94e018eb1d2f9e8217a88 2024-11-12T14:16:57,613 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6ea61e7c1d124f78e4a5241afdaab254/C of 6ea61e7c1d124f78e4a5241afdaab254 into 6b01852adec94e018eb1d2f9e8217a88(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:16:57,613 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:57,613 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., storeName=6ea61e7c1d124f78e4a5241afdaab254/C, priority=12, startTime=1731421017392; duration=0sec 2024-11-12T14:16:57,613 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:16:57,613 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ea61e7c1d124f78e4a5241afdaab254:C 2024-11-12T14:16:57,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:57,633 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ea61e7c1d124f78e4a5241afdaab254 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T14:16:57,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=A 2024-11-12T14:16:57,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:57,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=B 2024-11-12T14:16:57,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:57,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=C 2024-11-12T14:16:57,633 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:57,651 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/313a116602784593bf6501b37646d2df is 50, key is test_row_0/A:col10/1731421016704/Put/seqid=0 2024-11-12T14:16:57,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741915_1091 (size=14741) 2024-11-12T14:16:57,700 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:57,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421077696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:57,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:57,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:57,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:57,701 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:57,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421077699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:57,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:57,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:57,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:57,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/313a116602784593bf6501b37646d2df 2024-11-12T14:16:57,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/4b5dc9a124484a96a7f05f042f3c0382 is 50, key is test_row_0/B:col10/1731421016704/Put/seqid=0 2024-11-12T14:16:57,725 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741916_1092 (size=12301) 2024-11-12T14:16:57,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:57,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421077810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:57,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:57,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421077810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:57,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:57,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:57,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:57,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:57,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:57,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:57,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:57,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57718 deadline: 1731421077859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:57,861 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:57,861 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:57,861 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:57,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:57,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57778 deadline: 1731421077879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:57,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:57,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57768 deadline: 1731421077880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:57,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:57,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:57,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:57,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:57,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:57,883 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:58,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:58,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421078029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:58,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:58,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421078030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:58,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:58,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:58,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:58,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:58,032 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:58,032 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:58,038 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-12T14:16:58,038 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:16:58,038 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-12T14:16:58,039 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:16:58,041 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-12T14:16:58,042 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:16:58,044 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:16:58,044 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:16:58,045 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:16:58,045 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:16:58,045 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:16:58,045 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:16:58,046 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:16:58,046 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-12T14:16:58,046 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:16:58,046 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:16:58,046 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:16:58,046 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:16:58,046 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:16:58,048 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:16:58,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/4b5dc9a124484a96a7f05f042f3c0382 2024-11-12T14:16:58,135 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/79f7c61d06f947e68f61b49b05f7605a is 50, key is test_row_0/C:col10/1731421016704/Put/seqid=0 2024-11-12T14:16:58,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741917_1093 (size=12301) 2024-11-12T14:16:58,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-12T14:16:58,229 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:16:58,339 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:58,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:16:58,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57780 deadline: 1731421078339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:58,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57734 deadline: 1731421078339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:58,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:58,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:58,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:58,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:16:58,340 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:58,340 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:16:58,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=400 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/79f7c61d06f947e68f61b49b05f7605a 2024-11-12T14:16:58,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/313a116602784593bf6501b37646d2df as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/313a116602784593bf6501b37646d2df 2024-11-12T14:16:58,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/313a116602784593bf6501b37646d2df, entries=200, sequenceid=400, filesize=14.4 K 2024-11-12T14:16:58,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/4b5dc9a124484a96a7f05f042f3c0382 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/4b5dc9a124484a96a7f05f042f3c0382 2024-11-12T14:16:58,572 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/4b5dc9a124484a96a7f05f042f3c0382, entries=150, sequenceid=400, filesize=12.0 K 2024-11-12T14:16:58,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/79f7c61d06f947e68f61b49b05f7605a as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/79f7c61d06f947e68f61b49b05f7605a 2024-11-12T14:16:58,581 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/79f7c61d06f947e68f61b49b05f7605a, entries=150, sequenceid=400, filesize=12.0 K 2024-11-12T14:16:58,582 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 6ea61e7c1d124f78e4a5241afdaab254 in 949ms, sequenceid=400, compaction requested=false 2024-11-12T14:16:58,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:16:58,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:58,851 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6ea61e7c1d124f78e4a5241afdaab254 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T14:16:58,851 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-12T14:16:58,852 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-12T14:16:58,852 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:16:58,852 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:16:58,852 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:16:58,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=A 2024-11-12T14:16:58,852 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:16:58,852 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:16:58,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:58,852 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:16:58,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=B 2024-11-12T14:16:58,852 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:16:58,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:58,852 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:16:58,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=C 2024-11-12T14:16:58,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:16:58,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/bb145ba0a5994a18adbf68e90c876387 is 50, key is test_row_0/A:col10/1731421018850/Put/seqid=0 2024-11-12T14:16:58,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741918_1094 (size=12301) 2024-11-12T14:16:58,871 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-12T14:16:58,871 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:16:58,871 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:16:58,871 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:16:58,871 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:16:58,901 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-12T14:16:58,901 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-12T14:16:58,901 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:16:58,901 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:16:58,902 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:16:58,902 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:16:58,902 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:16:58,902 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:16:58,902 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:16:58,902 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:16:58,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T14:16:58,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-12T14:16:58,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 87 2024-11-12T14:16:58,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 77 2024-11-12T14:16:58,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-12T14:16:58,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-12T14:16:58,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T14:16:58,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1196 2024-11-12T14:16:58,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1112 2024-11-12T14:16:58,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T14:16:58,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 553 2024-11-12T14:16:58,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1652 rows 2024-11-12T14:16:58,902 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 565 2024-11-12T14:16:58,903 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 1687 rows 2024-11-12T14:16:58,903 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T14:16:58,903 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T14:16:58,903 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:16:58,903 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:16:58,903 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:16:58,906 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-12T14:16:58,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T14:16:58,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T14:16:58,913 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731421018912"}]},"ts":"1731421018912"} 2024-11-12T14:16:58,915 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T14:16:58,915 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T14:16:58,917 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T14:16:58,920 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T14:16:58,921 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ea61e7c1d124f78e4a5241afdaab254, UNASSIGN}] 2024-11-12T14:16:58,922 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ea61e7c1d124f78e4a5241afdaab254, UNASSIGN 2024-11-12T14:16:58,924 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=6ea61e7c1d124f78e4a5241afdaab254, regionState=CLOSING, regionLocation=2b6d221c5cde,42925,1731420990822 2024-11-12T14:16:58,926 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=23, ppid=22, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ea61e7c1d124f78e4a5241afdaab254, UNASSIGN because future has completed 2024-11-12T14:16:58,927 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T14:16:58,927 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822}] 2024-11-12T14:16:59,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T14:16:59,083 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(122): Close 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:16:59,083 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T14:16:59,084 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1722): Closing 6ea61e7c1d124f78e4a5241afdaab254, disabling compactions & flushes 2024-11-12T14:16:59,084 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:16:59,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T14:16:59,261 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/bb145ba0a5994a18adbf68e90c876387 2024-11-12T14:16:59,275 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/0928fc524ea74b5a8e2fbd359ff76348 is 50, key is test_row_0/B:col10/1731421018850/Put/seqid=0 2024-11-12T14:16:59,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741919_1095 (size=12301) 2024-11-12T14:16:59,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T14:16:59,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/0928fc524ea74b5a8e2fbd359ff76348 2024-11-12T14:16:59,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/b2f43f0ceff04fbd83babc21e56a7ba0 is 50, key is test_row_0/C:col10/1731421018850/Put/seqid=0 2024-11-12T14:16:59,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741920_1096 (size=12301) 2024-11-12T14:17:00,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T14:17:00,095 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=417 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/b2f43f0ceff04fbd83babc21e56a7ba0 2024-11-12T14:17:00,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/bb145ba0a5994a18adbf68e90c876387 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/bb145ba0a5994a18adbf68e90c876387 2024-11-12T14:17:00,106 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/bb145ba0a5994a18adbf68e90c876387, entries=150, sequenceid=417, filesize=12.0 K 2024-11-12T14:17:00,107 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/0928fc524ea74b5a8e2fbd359ff76348 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/0928fc524ea74b5a8e2fbd359ff76348 2024-11-12T14:17:00,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/0928fc524ea74b5a8e2fbd359ff76348, entries=150, sequenceid=417, filesize=12.0 K 2024-11-12T14:17:00,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/b2f43f0ceff04fbd83babc21e56a7ba0 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/b2f43f0ceff04fbd83babc21e56a7ba0 2024-11-12T14:17:00,119 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/b2f43f0ceff04fbd83babc21e56a7ba0, entries=150, sequenceid=417, filesize=12.0 K 2024-11-12T14:17:00,120 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=20.13 KB/20610 for 6ea61e7c1d124f78e4a5241afdaab254 in 1269ms, sequenceid=417, compaction requested=true 2024-11-12T14:17:00,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6ea61e7c1d124f78e4a5241afdaab254: 2024-11-12T14:17:00,120 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:17:00,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ea61e7c1d124f78e4a5241afdaab254:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:17:00,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:00,120 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:17:00,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ea61e7c1d124f78e4a5241afdaab254:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:17:00,120 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. after waiting 0 ms 2024-11-12T14:17:00,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:00,120 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:17:00,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6ea61e7c1d124f78e4a5241afdaab254:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:17:00,120 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. because compaction request was cancelled 2024-11-12T14:17:00,120 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:00,120 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ea61e7c1d124f78e4a5241afdaab254:A 2024-11-12T14:17:00,120 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. because compaction request was cancelled 2024-11-12T14:17:00,121 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(2902): Flushing 6ea61e7c1d124f78e4a5241afdaab254 3/3 column families, dataSize=20.13 KB heapSize=60.52 KB 2024-11-12T14:17:00,121 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ea61e7c1d124f78e4a5241afdaab254:C 2024-11-12T14:17:00,121 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. because compaction request was cancelled 2024-11-12T14:17:00,121 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6ea61e7c1d124f78e4a5241afdaab254:B 2024-11-12T14:17:00,121 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=A 2024-11-12T14:17:00,121 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:00,121 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=B 2024-11-12T14:17:00,121 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:00,121 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 6ea61e7c1d124f78e4a5241afdaab254, store=C 2024-11-12T14:17:00,121 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:00,125 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/f4bdd7714eab4444b753cd43774cae3f is 50, key is test_row_0/A:col10/1731421018900/Put/seqid=0 2024-11-12T14:17:00,130 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741921_1097 (size=12301) 2024-11-12T14:17:00,531 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/f4bdd7714eab4444b753cd43774cae3f 2024-11-12T14:17:00,538 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/67ef3fb21fab4b5993d0854ea193b741 is 50, key is test_row_0/B:col10/1731421018900/Put/seqid=0 2024-11-12T14:17:00,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741922_1098 (size=12301) 2024-11-12T14:17:00,943 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/67ef3fb21fab4b5993d0854ea193b741 2024-11-12T14:17:00,954 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/9c4d94636130447498a92a0b604d6860 is 50, key is test_row_0/C:col10/1731421018900/Put/seqid=0 2024-11-12T14:17:00,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741923_1099 (size=12301) 2024-11-12T14:17:00,973 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/9c4d94636130447498a92a0b604d6860 2024-11-12T14:17:00,983 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/A/f4bdd7714eab4444b753cd43774cae3f as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/f4bdd7714eab4444b753cd43774cae3f 2024-11-12T14:17:00,990 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/f4bdd7714eab4444b753cd43774cae3f, entries=150, sequenceid=423, filesize=12.0 K 2024-11-12T14:17:00,992 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/B/67ef3fb21fab4b5993d0854ea193b741 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/67ef3fb21fab4b5993d0854ea193b741 2024-11-12T14:17:00,999 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/67ef3fb21fab4b5993d0854ea193b741, entries=150, sequenceid=423, filesize=12.0 K 2024-11-12T14:17:01,000 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/.tmp/C/9c4d94636130447498a92a0b604d6860 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/9c4d94636130447498a92a0b604d6860 2024-11-12T14:17:01,007 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/9c4d94636130447498a92a0b604d6860, entries=150, sequenceid=423, filesize=12.0 K 2024-11-12T14:17:01,008 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 6ea61e7c1d124f78e4a5241afdaab254 in 888ms, sequenceid=423, compaction requested=true 2024-11-12T14:17:01,009 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/08feba09c95c43b484e9b2d0ab6fb503, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/bae66fce00154e688a47fb8355495de9, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/ea995fa4ab334484805f29697ee45978, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/89690dae6c5d4aebabbfb9d8e7fa753b, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/2c67e6f5e065492f8e61dbf29f7ccfed, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/6c70741c43f94833a3f8c30a0cd12564, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/5422e64db8244b6fa12dd510e8660d4c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/461922825eaf46aca28b9b7f40945202, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/5d0e9dd38ea6449585fc49882de5d331, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/b8baffd54d25404f914dd772c83182cc, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/9961ff4a421a48fcaf6673d6350b5664, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/503976c02e694e2182bab60d8505a9d1, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/d21d5428f2f24845b6e4dd7fa22e1c73, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/ccfc6d3beaf144679f45b7eaa984aded, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/b754eeb71830426fb7f9e21aaa5712cf, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/6ceaf1c6215d4f8185505e5bcee4529f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/2fa4df8bdce8445b9d2e135f05316d9a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/8ec7d3b356ac40049193d1a6a1c21f9a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/0986dd238b1f4feb8a6074d6759709bb, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/0de10473ad744c689d11c8faad9e8440, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/952ac7d7622743bfa96eecbfc30debce, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/cd12081b257d405cb9251aac257d3980, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/222bba6d92604c00ba1a54e8455154f7, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/8ebf6365bb504a40ab199728e5b16a6e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/7bfd4295ce48402792e82e65108729a8] to archive 2024-11-12T14:17:01,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T14:17:01,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/08feba09c95c43b484e9b2d0ab6fb503 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/08feba09c95c43b484e9b2d0ab6fb503 2024-11-12T14:17:01,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/bae66fce00154e688a47fb8355495de9 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/bae66fce00154e688a47fb8355495de9 2024-11-12T14:17:01,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/ea995fa4ab334484805f29697ee45978 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/ea995fa4ab334484805f29697ee45978 2024-11-12T14:17:01,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/89690dae6c5d4aebabbfb9d8e7fa753b to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/89690dae6c5d4aebabbfb9d8e7fa753b 2024-11-12T14:17:01,054 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/2c67e6f5e065492f8e61dbf29f7ccfed to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/2c67e6f5e065492f8e61dbf29f7ccfed 2024-11-12T14:17:01,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/6c70741c43f94833a3f8c30a0cd12564 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/6c70741c43f94833a3f8c30a0cd12564 2024-11-12T14:17:01,061 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/5422e64db8244b6fa12dd510e8660d4c to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/5422e64db8244b6fa12dd510e8660d4c 2024-11-12T14:17:01,063 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/461922825eaf46aca28b9b7f40945202 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/461922825eaf46aca28b9b7f40945202 2024-11-12T14:17:01,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/5d0e9dd38ea6449585fc49882de5d331 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/5d0e9dd38ea6449585fc49882de5d331 2024-11-12T14:17:01,067 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/b8baffd54d25404f914dd772c83182cc to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/b8baffd54d25404f914dd772c83182cc 2024-11-12T14:17:01,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T14:17:01,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/9961ff4a421a48fcaf6673d6350b5664 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/9961ff4a421a48fcaf6673d6350b5664 2024-11-12T14:17:01,072 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/503976c02e694e2182bab60d8505a9d1 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/503976c02e694e2182bab60d8505a9d1 2024-11-12T14:17:01,075 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/d21d5428f2f24845b6e4dd7fa22e1c73 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/d21d5428f2f24845b6e4dd7fa22e1c73 2024-11-12T14:17:01,077 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/ccfc6d3beaf144679f45b7eaa984aded to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/ccfc6d3beaf144679f45b7eaa984aded 2024-11-12T14:17:01,081 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/b754eeb71830426fb7f9e21aaa5712cf to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/b754eeb71830426fb7f9e21aaa5712cf 2024-11-12T14:17:01,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/6ceaf1c6215d4f8185505e5bcee4529f to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/6ceaf1c6215d4f8185505e5bcee4529f 2024-11-12T14:17:01,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/2fa4df8bdce8445b9d2e135f05316d9a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/2fa4df8bdce8445b9d2e135f05316d9a 2024-11-12T14:17:01,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/8ec7d3b356ac40049193d1a6a1c21f9a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/8ec7d3b356ac40049193d1a6a1c21f9a 2024-11-12T14:17:01,099 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/0986dd238b1f4feb8a6074d6759709bb to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/0986dd238b1f4feb8a6074d6759709bb 2024-11-12T14:17:01,101 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/0de10473ad744c689d11c8faad9e8440 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/0de10473ad744c689d11c8faad9e8440 2024-11-12T14:17:01,103 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/952ac7d7622743bfa96eecbfc30debce to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/952ac7d7622743bfa96eecbfc30debce 2024-11-12T14:17:01,116 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/cd12081b257d405cb9251aac257d3980 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/cd12081b257d405cb9251aac257d3980 2024-11-12T14:17:01,123 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/222bba6d92604c00ba1a54e8455154f7 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/222bba6d92604c00ba1a54e8455154f7 2024-11-12T14:17:01,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/8ebf6365bb504a40ab199728e5b16a6e to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/8ebf6365bb504a40ab199728e5b16a6e 2024-11-12T14:17:01,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/7bfd4295ce48402792e82e65108729a8 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/7bfd4295ce48402792e82e65108729a8 2024-11-12T14:17:01,149 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/44505f337484442da4f854bbab0a652b, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/b7ddf445cfc04660882023922d42e0d7, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/760c9b2620234350b0efc51a32e3e801, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/06eac540edb0487091891ee6f3d5846f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/8a39f03e2d8f48a3beaa8c857f3dfcaa, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/fba519f9cffa489595ce24b603522709, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/c3cda5212db34baa9fffb5dcef511632, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/8a19c7d41efc44c593cfce707c3a1fb9, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/f1c87643e3db4ff58b211ca4ee173984, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/67d7280b788142d1acff8c921423ed7e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/6711321620c34778ac87e055f42c589f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/15729b4f1eef4a05b2494144594a77e7, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/435d7cfdb6474d18859d44263c84fac5, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/374e05b3346940aeb1a34ce699c567e6, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/9feee99b998747f5a286d5670f1323a2, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/f0dcfa9d79ac405ca4d52a0d444e7fa6, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/2d97ed30968248179aa09d120c3d96bf, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/386ab9e3fc614d43829a1fdbd3caae75, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/f97cfc95673349e78b1a5dba9476e039, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/d91ced07fee340a8a5e5cd6ff275a024, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/33944f5e65eb4d15960ef06d0c847300, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/48c163760f3a4366aeac534fc06c89a3, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/4b6a3530f9904a09b66c2e17ff31ae0c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/a1068de709534f249eaa55254d0a2d1c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/e7c73010aaa24626861671db2e006de0] to archive 2024-11-12T14:17:01,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T14:17:01,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/44505f337484442da4f854bbab0a652b to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/44505f337484442da4f854bbab0a652b 2024-11-12T14:17:01,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/b7ddf445cfc04660882023922d42e0d7 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/b7ddf445cfc04660882023922d42e0d7 2024-11-12T14:17:01,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/760c9b2620234350b0efc51a32e3e801 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/760c9b2620234350b0efc51a32e3e801 2024-11-12T14:17:01,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/06eac540edb0487091891ee6f3d5846f to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/06eac540edb0487091891ee6f3d5846f 2024-11-12T14:17:01,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/8a39f03e2d8f48a3beaa8c857f3dfcaa to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/8a39f03e2d8f48a3beaa8c857f3dfcaa 2024-11-12T14:17:01,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/fba519f9cffa489595ce24b603522709 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/fba519f9cffa489595ce24b603522709 2024-11-12T14:17:01,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/c3cda5212db34baa9fffb5dcef511632 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/c3cda5212db34baa9fffb5dcef511632 2024-11-12T14:17:01,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/8a19c7d41efc44c593cfce707c3a1fb9 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/8a19c7d41efc44c593cfce707c3a1fb9 2024-11-12T14:17:01,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/f1c87643e3db4ff58b211ca4ee173984 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/f1c87643e3db4ff58b211ca4ee173984 2024-11-12T14:17:01,212 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/67d7280b788142d1acff8c921423ed7e to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/67d7280b788142d1acff8c921423ed7e 2024-11-12T14:17:01,215 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/6711321620c34778ac87e055f42c589f to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/6711321620c34778ac87e055f42c589f 2024-11-12T14:17:01,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/15729b4f1eef4a05b2494144594a77e7 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/15729b4f1eef4a05b2494144594a77e7 2024-11-12T14:17:01,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/435d7cfdb6474d18859d44263c84fac5 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/435d7cfdb6474d18859d44263c84fac5 2024-11-12T14:17:01,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/374e05b3346940aeb1a34ce699c567e6 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/374e05b3346940aeb1a34ce699c567e6 2024-11-12T14:17:01,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/9feee99b998747f5a286d5670f1323a2 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/9feee99b998747f5a286d5670f1323a2 2024-11-12T14:17:01,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/f0dcfa9d79ac405ca4d52a0d444e7fa6 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/f0dcfa9d79ac405ca4d52a0d444e7fa6 2024-11-12T14:17:01,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/2d97ed30968248179aa09d120c3d96bf to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/2d97ed30968248179aa09d120c3d96bf 2024-11-12T14:17:01,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/386ab9e3fc614d43829a1fdbd3caae75 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/386ab9e3fc614d43829a1fdbd3caae75 2024-11-12T14:17:01,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/f97cfc95673349e78b1a5dba9476e039 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/f97cfc95673349e78b1a5dba9476e039 2024-11-12T14:17:01,272 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/d91ced07fee340a8a5e5cd6ff275a024 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/d91ced07fee340a8a5e5cd6ff275a024 2024-11-12T14:17:01,278 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/33944f5e65eb4d15960ef06d0c847300 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/33944f5e65eb4d15960ef06d0c847300 2024-11-12T14:17:01,284 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/48c163760f3a4366aeac534fc06c89a3 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/48c163760f3a4366aeac534fc06c89a3 2024-11-12T14:17:01,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/4b6a3530f9904a09b66c2e17ff31ae0c to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/4b6a3530f9904a09b66c2e17ff31ae0c 2024-11-12T14:17:01,291 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/a1068de709534f249eaa55254d0a2d1c to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/a1068de709534f249eaa55254d0a2d1c 2024-11-12T14:17:01,292 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/e7c73010aaa24626861671db2e006de0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/e7c73010aaa24626861671db2e006de0 2024-11-12T14:17:01,296 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/ae7f1e5c1b90408cb4aa1f895ae2e9d4, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/8bcf3c54b8204ef6a8d9ffe1f140ec5a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/12a4d09d0c1e43d2b02b0b0171850b7d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/74fafa2273084d7c9adfca7a4b8efa30, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/c997088aa181483eba1eaa405ebc8fd0, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/7e495ffce5644be8a35d47b5bd4507b7, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/0a7806d6a19243d3ab57c6240bacd0dc, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/8b2517ec1ce74432b0c6f376876e120a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/a46b881be4224f7995b73d9ad2276353, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/66beeb7f86404bcd97fe236f558678ee, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/0b27d952afda42a2bdcab96d2d0ba1d7, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/a037613c9dac4b6ab9fb675c81211c75, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/297ffb33b2964320b29c315ca171b6de, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/d04fe8ccd68e4174a71711348ad65b71, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/d1244190df6840948e3a846e9e7dde13, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/0a4acbca7d044fde99b4ce4deee0500f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/6228b841b5324d24b099bbdcb9678fcb, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/72f3ec3f489e413c8c3209dfa3ab1ae9, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/67bf68ab221340f8a04d5b8acb428a9f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/7f0d78109eac4c32a9bd155cd077fc22, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/a01461c9f6724f9ca5fc0492c5329aff, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/5cba51deba6640e0bfe664fe29f2ae84, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/9f0279c92ff543ca9e77b880208d6ca2, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/047fa94674c6411d8a914f84d13ebe19, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/69c6010737f04c50b8f752eb48bcc8e2] to archive 2024-11-12T14:17:01,298 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T14:17:01,307 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/ae7f1e5c1b90408cb4aa1f895ae2e9d4 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/ae7f1e5c1b90408cb4aa1f895ae2e9d4 2024-11-12T14:17:01,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/8bcf3c54b8204ef6a8d9ffe1f140ec5a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/8bcf3c54b8204ef6a8d9ffe1f140ec5a 2024-11-12T14:17:01,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/12a4d09d0c1e43d2b02b0b0171850b7d to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/12a4d09d0c1e43d2b02b0b0171850b7d 2024-11-12T14:17:01,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/74fafa2273084d7c9adfca7a4b8efa30 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/74fafa2273084d7c9adfca7a4b8efa30 2024-11-12T14:17:01,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/c997088aa181483eba1eaa405ebc8fd0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/c997088aa181483eba1eaa405ebc8fd0 2024-11-12T14:17:01,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/7e495ffce5644be8a35d47b5bd4507b7 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/7e495ffce5644be8a35d47b5bd4507b7 2024-11-12T14:17:01,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/0a7806d6a19243d3ab57c6240bacd0dc to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/0a7806d6a19243d3ab57c6240bacd0dc 2024-11-12T14:17:01,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/8b2517ec1ce74432b0c6f376876e120a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/8b2517ec1ce74432b0c6f376876e120a 2024-11-12T14:17:01,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/a46b881be4224f7995b73d9ad2276353 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/a46b881be4224f7995b73d9ad2276353 2024-11-12T14:17:01,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/66beeb7f86404bcd97fe236f558678ee to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/66beeb7f86404bcd97fe236f558678ee 2024-11-12T14:17:01,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/0b27d952afda42a2bdcab96d2d0ba1d7 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/0b27d952afda42a2bdcab96d2d0ba1d7 2024-11-12T14:17:01,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/a037613c9dac4b6ab9fb675c81211c75 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/a037613c9dac4b6ab9fb675c81211c75 2024-11-12T14:17:01,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/297ffb33b2964320b29c315ca171b6de to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/297ffb33b2964320b29c315ca171b6de 2024-11-12T14:17:01,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/d04fe8ccd68e4174a71711348ad65b71 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/d04fe8ccd68e4174a71711348ad65b71 2024-11-12T14:17:01,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/d1244190df6840948e3a846e9e7dde13 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/d1244190df6840948e3a846e9e7dde13 2024-11-12T14:17:01,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/0a4acbca7d044fde99b4ce4deee0500f to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/0a4acbca7d044fde99b4ce4deee0500f 2024-11-12T14:17:01,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/6228b841b5324d24b099bbdcb9678fcb to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/6228b841b5324d24b099bbdcb9678fcb 2024-11-12T14:17:01,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/72f3ec3f489e413c8c3209dfa3ab1ae9 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/72f3ec3f489e413c8c3209dfa3ab1ae9 2024-11-12T14:17:01,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/67bf68ab221340f8a04d5b8acb428a9f to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/67bf68ab221340f8a04d5b8acb428a9f 2024-11-12T14:17:01,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/7f0d78109eac4c32a9bd155cd077fc22 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/7f0d78109eac4c32a9bd155cd077fc22 2024-11-12T14:17:01,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/a01461c9f6724f9ca5fc0492c5329aff to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/a01461c9f6724f9ca5fc0492c5329aff 2024-11-12T14:17:01,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/5cba51deba6640e0bfe664fe29f2ae84 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/5cba51deba6640e0bfe664fe29f2ae84 2024-11-12T14:17:01,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/9f0279c92ff543ca9e77b880208d6ca2 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/9f0279c92ff543ca9e77b880208d6ca2 2024-11-12T14:17:01,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/047fa94674c6411d8a914f84d13ebe19 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/047fa94674c6411d8a914f84d13ebe19 2024-11-12T14:17:01,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/69c6010737f04c50b8f752eb48bcc8e2 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/69c6010737f04c50b8f752eb48bcc8e2 2024-11-12T14:17:01,388 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/recovered.edits/426.seqid, newMaxSeqId=426, maxSeqId=1 2024-11-12T14:17:01,393 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. 2024-11-12T14:17:01,393 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] regionserver.HRegion(1676): Region close journal for 6ea61e7c1d124f78e4a5241afdaab254: Waiting for close lock at 1731421019084Running coprocessor pre-close hooks at 1731421019084Disabling compacts and flushes for region at 1731421019084Disabling writes for close at 1731421020120 (+1036 ms)Obtaining lock to block concurrent updates at 1731421020121 (+1 ms)Preparing flush snapshotting stores in 6ea61e7c1d124f78e4a5241afdaab254 at 1731421020121Finished memstore snapshotting TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=61920, getOffHeapSize=0, getCellsCount=450 at 1731421020121Flushing stores of TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254. at 1731421020122 (+1 ms)Flushing 6ea61e7c1d124f78e4a5241afdaab254/A: creating writer at 1731421020122Flushing 6ea61e7c1d124f78e4a5241afdaab254/A: appending metadata at 1731421020125 (+3 ms)Flushing 6ea61e7c1d124f78e4a5241afdaab254/A: closing flushed file at 1731421020125Flushing 6ea61e7c1d124f78e4a5241afdaab254/B: creating writer at 1731421020535 (+410 ms)Flushing 6ea61e7c1d124f78e4a5241afdaab254/B: appending metadata at 1731421020538 (+3 ms)Flushing 6ea61e7c1d124f78e4a5241afdaab254/B: closing flushed file at 1731421020538Flushing 6ea61e7c1d124f78e4a5241afdaab254/C: creating writer at 1731421020950 (+412 ms)Flushing 6ea61e7c1d124f78e4a5241afdaab254/C: appending metadata at 1731421020954 (+4 ms)Flushing 6ea61e7c1d124f78e4a5241afdaab254/C: closing flushed file at 1731421020954Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@56f14796: reopening flushed file at 1731421020982 (+28 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5eb1543: reopening flushed file at 1731421020991 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40c7b2c2: reopening flushed file at 1731421020999 (+8 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~60.47 KB/61920, currentSize=0 B/0 for 6ea61e7c1d124f78e4a5241afdaab254 in 888ms, sequenceid=423, compaction requested=true at 1731421021009 (+10 ms)Writing region close event to WAL at 1731421021384 (+375 ms)Running coprocessor post-close hooks at 1731421021389 (+5 ms)Closed at 1731421021393 (+4 ms) 2024-11-12T14:17:01,399 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=23 updating hbase:meta row=6ea61e7c1d124f78e4a5241afdaab254, regionState=CLOSED 2024-11-12T14:17:01,399 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=24}] handler.UnassignRegionHandler(157): Closed 6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:17:01,401 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=24, ppid=23, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 because future has completed 2024-11-12T14:17:01,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-12T14:17:01,404 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; CloseRegionProcedure 6ea61e7c1d124f78e4a5241afdaab254, server=2b6d221c5cde,42925,1731420990822 in 2.4750 sec 2024-11-12T14:17:01,406 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=23, resume processing ppid=22 2024-11-12T14:17:01,406 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, ppid=22, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6ea61e7c1d124f78e4a5241afdaab254, UNASSIGN in 2.4830 sec 2024-11-12T14:17:01,409 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-12T14:17:01,409 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.4910 sec 2024-11-12T14:17:01,411 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731421021411"}]},"ts":"1731421021411"} 2024-11-12T14:17:01,415 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T14:17:01,415 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T14:17:01,419 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.5090 sec 2024-11-12T14:17:03,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-12T14:17:03,090 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:17:03,093 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-12T14:17:03,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:17:03,098 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=25, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:17:03,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-12T14:17:03,100 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=25, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:17:03,105 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:17:03,109 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A, FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B, FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C, FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/recovered.edits] 2024-11-12T14:17:03,112 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/313a116602784593bf6501b37646d2df to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/313a116602784593bf6501b37646d2df 2024-11-12T14:17:03,115 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/602078873a0149ffbd71f7b2f607270e to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/602078873a0149ffbd71f7b2f607270e 2024-11-12T14:17:03,117 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/bb145ba0a5994a18adbf68e90c876387 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/bb145ba0a5994a18adbf68e90c876387 2024-11-12T14:17:03,118 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/f4bdd7714eab4444b753cd43774cae3f to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/A/f4bdd7714eab4444b753cd43774cae3f 2024-11-12T14:17:03,122 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/0928fc524ea74b5a8e2fbd359ff76348 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/0928fc524ea74b5a8e2fbd359ff76348 2024-11-12T14:17:03,123 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/4b5dc9a124484a96a7f05f042f3c0382 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/4b5dc9a124484a96a7f05f042f3c0382 2024-11-12T14:17:03,126 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/67ef3fb21fab4b5993d0854ea193b741 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/67ef3fb21fab4b5993d0854ea193b741 2024-11-12T14:17:03,129 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/75c06f410d6044689e0e3cedb9307d3f to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/B/75c06f410d6044689e0e3cedb9307d3f 2024-11-12T14:17:03,134 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/6b01852adec94e018eb1d2f9e8217a88 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/6b01852adec94e018eb1d2f9e8217a88 2024-11-12T14:17:03,136 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/79f7c61d06f947e68f61b49b05f7605a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/79f7c61d06f947e68f61b49b05f7605a 2024-11-12T14:17:03,137 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/9c4d94636130447498a92a0b604d6860 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/9c4d94636130447498a92a0b604d6860 2024-11-12T14:17:03,140 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/b2f43f0ceff04fbd83babc21e56a7ba0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/C/b2f43f0ceff04fbd83babc21e56a7ba0 2024-11-12T14:17:03,145 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/recovered.edits/426.seqid to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254/recovered.edits/426.seqid 2024-11-12T14:17:03,146 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/6ea61e7c1d124f78e4a5241afdaab254 2024-11-12T14:17:03,146 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T14:17:03,152 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=25, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:17:03,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-12T14:17:03,163 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T14:17:03,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-12T14:17:03,210 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T14:17:03,213 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=25, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:17:03,214 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T14:17:03,214 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731421023214"}]},"ts":"9223372036854775807"} 2024-11-12T14:17:03,220 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T14:17:03,220 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6ea61e7c1d124f78e4a5241afdaab254, NAME => 'TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T14:17:03,220 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T14:17:03,221 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731421023220"}]},"ts":"9223372036854775807"} 2024-11-12T14:17:03,225 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T14:17:03,229 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=25, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:17:03,231 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 136 msec 2024-11-12T14:17:03,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-12T14:17:03,420 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T14:17:03,421 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:17:03,450 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMixedAtomicity Thread=249 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-8 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1315096866_22 at /127.0.0.1:57436 [Waiting for operation #154] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;2b6d221c5cde:42925-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1274387348_22 at /127.0.0.1:39916 [Waiting for operation #186] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-9 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1315096866_22 at /127.0.0.1:54860 [Waiting for operation #199] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1082 (was 813) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1069 (was 2651) 2024-11-12T14:17:03,473 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=249, OpenFileDescriptor=457, MaxFileDescriptor=1048576, SystemLoadAverage=1082, ProcessCount=11, AvailableMemoryMB=1068 2024-11-12T14:17:03,476 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T14:17:03,477 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T14:17:03,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T14:17:03,480 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T14:17:03,480 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:03,480 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 26 2024-11-12T14:17:03,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-12T14:17:03,482 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T14:17:03,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741924_1100 (size=1061) 2024-11-12T14:17:03,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-12T14:17:03,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-12T14:17:03,970 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3 2024-11-12T14:17:04,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741925_1101 (size=53) 2024-11-12T14:17:04,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-12T14:17:04,413 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:17:04,413 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 1a0d122b7a4f4891082438eedd4480d0, disabling compactions & flushes 2024-11-12T14:17:04,413 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:04,413 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:04,414 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. after waiting 0 ms 2024-11-12T14:17:04,414 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:04,414 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:04,414 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 1a0d122b7a4f4891082438eedd4480d0: Waiting for close lock at 1731421024413Disabling compacts and flushes for region at 1731421024413Disabling writes for close at 1731421024414 (+1 ms)Writing region close event to WAL at 1731421024414Closed at 1731421024414 2024-11-12T14:17:04,416 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T14:17:04,417 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731421024416"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731421024416"}]},"ts":"1731421024416"} 2024-11-12T14:17:04,424 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T14:17:04,426 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T14:17:04,426 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731421024426"}]},"ts":"1731421024426"} 2024-11-12T14:17:04,429 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T14:17:04,429 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a0d122b7a4f4891082438eedd4480d0, ASSIGN}] 2024-11-12T14:17:04,431 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a0d122b7a4f4891082438eedd4480d0, ASSIGN 2024-11-12T14:17:04,434 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a0d122b7a4f4891082438eedd4480d0, ASSIGN; state=OFFLINE, location=2b6d221c5cde,42925,1731420990822; forceNewPlan=false, retain=false 2024-11-12T14:17:04,589 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=1a0d122b7a4f4891082438eedd4480d0, regionState=OPENING, regionLocation=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:04,591 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=27, ppid=26, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a0d122b7a4f4891082438eedd4480d0, ASSIGN because future has completed 2024-11-12T14:17:04,592 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822}] 2024-11-12T14:17:04,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-12T14:17:04,780 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:04,780 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7752): Opening region: {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:17:04,781 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:04,781 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:17:04,781 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7794): checking encryption for 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:04,781 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(7797): checking classloading for 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:04,796 INFO [StoreOpener-1a0d122b7a4f4891082438eedd4480d0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:04,798 INFO [StoreOpener-1a0d122b7a4f4891082438eedd4480d0-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T14:17:04,800 INFO [StoreOpener-1a0d122b7a4f4891082438eedd4480d0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a0d122b7a4f4891082438eedd4480d0 columnFamilyName A 2024-11-12T14:17:04,800 DEBUG [StoreOpener-1a0d122b7a4f4891082438eedd4480d0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:04,802 INFO [StoreOpener-1a0d122b7a4f4891082438eedd4480d0-1 {}] regionserver.HStore(327): Store=1a0d122b7a4f4891082438eedd4480d0/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:17:04,802 INFO [StoreOpener-1a0d122b7a4f4891082438eedd4480d0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:04,804 INFO [StoreOpener-1a0d122b7a4f4891082438eedd4480d0-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T14:17:04,804 INFO [StoreOpener-1a0d122b7a4f4891082438eedd4480d0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a0d122b7a4f4891082438eedd4480d0 columnFamilyName B 2024-11-12T14:17:04,804 DEBUG [StoreOpener-1a0d122b7a4f4891082438eedd4480d0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:04,805 INFO [StoreOpener-1a0d122b7a4f4891082438eedd4480d0-1 {}] regionserver.HStore(327): Store=1a0d122b7a4f4891082438eedd4480d0/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:17:04,806 INFO [StoreOpener-1a0d122b7a4f4891082438eedd4480d0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:04,808 INFO [StoreOpener-1a0d122b7a4f4891082438eedd4480d0-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T14:17:04,808 INFO [StoreOpener-1a0d122b7a4f4891082438eedd4480d0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a0d122b7a4f4891082438eedd4480d0 columnFamilyName C 2024-11-12T14:17:04,808 DEBUG [StoreOpener-1a0d122b7a4f4891082438eedd4480d0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:04,809 INFO [StoreOpener-1a0d122b7a4f4891082438eedd4480d0-1 {}] regionserver.HStore(327): Store=1a0d122b7a4f4891082438eedd4480d0/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:17:04,809 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:04,809 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1038): replaying wal for 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:04,810 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:04,810 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:04,811 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1048): stopping wal replay for 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:04,812 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1060): Cleaning up temporary data for 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:04,812 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T14:17:04,817 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1093): writing seq id for 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:04,820 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:17:04,821 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1114): Opened 1a0d122b7a4f4891082438eedd4480d0; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59377334, jitterRate=-0.11520877480506897}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T14:17:04,821 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:04,822 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegion(1006): Region open journal for 1a0d122b7a4f4891082438eedd4480d0: Running coprocessor pre-open hook at 1731421024781Writing region info on filesystem at 1731421024781Initializing all the Stores at 1731421024786 (+5 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731421024786Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731421024796 (+10 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731421024796Cleaning up temporary data from old regions at 1731421024812 (+16 ms)Running coprocessor post-open hooks at 1731421024821 (+9 ms)Region opened successfully at 1731421024822 (+1 ms) 2024-11-12T14:17:04,828 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., pid=28, masterSystemTime=1731421024756 2024-11-12T14:17:04,843 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:04,843 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=28}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:04,843 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=27 updating hbase:meta row=1a0d122b7a4f4891082438eedd4480d0, regionState=OPEN, openSeqNum=2, regionLocation=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:04,852 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=28, ppid=27, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 because future has completed 2024-11-12T14:17:04,854 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45591 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=2b6d221c5cde,42925,1731420990822, table=TestAcidGuarantees, region=1a0d122b7a4f4891082438eedd4480d0. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T14:17:04,863 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-12T14:17:04,864 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; OpenRegionProcedure 1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 in 265 msec 2024-11-12T14:17:04,869 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T14:17:04,870 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731421024869"}]},"ts":"1731421024869"} 2024-11-12T14:17:04,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=27, resume processing ppid=26 2024-11-12T14:17:04,870 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, ppid=26, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a0d122b7a4f4891082438eedd4480d0, ASSIGN in 435 msec 2024-11-12T14:17:04,872 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T14:17:04,873 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=26, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T14:17:04,876 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.3960 sec 2024-11-12T14:17:05,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=26 2024-11-12T14:17:05,630 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:17:05,631 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c8fb52a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:05,631 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:17:05,631 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:17:05,633 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:17:05,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:17:05,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:17:05,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37a15ec4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:05,633 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:17:05,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:17:05,634 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:05,636 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55282, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:17:05,637 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12afde2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:05,637 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:17:05,638 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:17:05,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:17:05,640 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42892, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:17:05,642 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T14:17:05,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 2b6d221c5cde,45591,1731420989939 2024-11-12T14:17:05,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@302feb9a 2024-11-12T14:17:05,643 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T14:17:05,645 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55290, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T14:17:05,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T14:17:05,651 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T14:17:05,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-12T14:17:05,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741926_1102 (size=1097) 2024-11-12T14:17:05,680 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-12T14:17:05,680 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-12T14:17:05,685 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T14:17:05,701 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a0d122b7a4f4891082438eedd4480d0, REOPEN/MOVE}] 2024-11-12T14:17:05,704 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a0d122b7a4f4891082438eedd4480d0, REOPEN/MOVE 2024-11-12T14:17:05,705 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=1a0d122b7a4f4891082438eedd4480d0, regionState=CLOSING, regionLocation=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:05,706 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a0d122b7a4f4891082438eedd4480d0, REOPEN/MOVE because future has completed 2024-11-12T14:17:05,708 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T14:17:05,708 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822}] 2024-11-12T14:17:05,861 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:05,861 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T14:17:05,861 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 1a0d122b7a4f4891082438eedd4480d0, disabling compactions & flushes 2024-11-12T14:17:05,861 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:05,861 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:05,861 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. after waiting 0 ms 2024-11-12T14:17:05,861 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:05,868 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T14:17:05,869 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:05,869 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 1a0d122b7a4f4891082438eedd4480d0: Waiting for close lock at 1731421025861Running coprocessor pre-close hooks at 1731421025861Disabling compacts and flushes for region at 1731421025861Disabling writes for close at 1731421025861Writing region close event to WAL at 1731421025864 (+3 ms)Running coprocessor post-close hooks at 1731421025869 (+5 ms)Closed at 1731421025869 2024-11-12T14:17:05,870 WARN [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionServer(3299): Not adding moved region record: 1a0d122b7a4f4891082438eedd4480d0 to self. 2024-11-12T14:17:05,871 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:05,872 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=1a0d122b7a4f4891082438eedd4480d0, regionState=CLOSED 2024-11-12T14:17:05,875 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 because future has completed 2024-11-12T14:17:05,879 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-12T14:17:05,879 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 in 168 msec 2024-11-12T14:17:05,880 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a0d122b7a4f4891082438eedd4480d0, REOPEN/MOVE; state=CLOSED, location=2b6d221c5cde,42925,1731420990822; forceNewPlan=false, retain=true 2024-11-12T14:17:06,030 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=1a0d122b7a4f4891082438eedd4480d0, regionState=OPENING, regionLocation=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:06,035 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a0d122b7a4f4891082438eedd4480d0, REOPEN/MOVE because future has completed 2024-11-12T14:17:06,035 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822}] 2024-11-12T14:17:06,210 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:06,211 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7752): Opening region: {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:17:06,211 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:06,211 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:17:06,211 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7794): checking encryption for 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:06,211 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(7797): checking classloading for 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:06,220 INFO [StoreOpener-1a0d122b7a4f4891082438eedd4480d0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:06,220 INFO [StoreOpener-1a0d122b7a4f4891082438eedd4480d0-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T14:17:06,225 INFO [StoreOpener-1a0d122b7a4f4891082438eedd4480d0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a0d122b7a4f4891082438eedd4480d0 columnFamilyName A 2024-11-12T14:17:06,229 DEBUG [StoreOpener-1a0d122b7a4f4891082438eedd4480d0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:06,230 INFO [StoreOpener-1a0d122b7a4f4891082438eedd4480d0-1 {}] regionserver.HStore(327): Store=1a0d122b7a4f4891082438eedd4480d0/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:17:06,231 INFO [StoreOpener-1a0d122b7a4f4891082438eedd4480d0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:06,232 INFO [StoreOpener-1a0d122b7a4f4891082438eedd4480d0-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T14:17:06,232 INFO [StoreOpener-1a0d122b7a4f4891082438eedd4480d0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a0d122b7a4f4891082438eedd4480d0 columnFamilyName B 2024-11-12T14:17:06,232 DEBUG [StoreOpener-1a0d122b7a4f4891082438eedd4480d0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:06,233 INFO [StoreOpener-1a0d122b7a4f4891082438eedd4480d0-1 {}] regionserver.HStore(327): Store=1a0d122b7a4f4891082438eedd4480d0/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:17:06,233 INFO [StoreOpener-1a0d122b7a4f4891082438eedd4480d0-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:06,233 INFO [StoreOpener-1a0d122b7a4f4891082438eedd4480d0-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T14:17:06,234 INFO [StoreOpener-1a0d122b7a4f4891082438eedd4480d0-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1a0d122b7a4f4891082438eedd4480d0 columnFamilyName C 2024-11-12T14:17:06,234 DEBUG [StoreOpener-1a0d122b7a4f4891082438eedd4480d0-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:06,235 INFO [StoreOpener-1a0d122b7a4f4891082438eedd4480d0-1 {}] regionserver.HStore(327): Store=1a0d122b7a4f4891082438eedd4480d0/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:17:06,235 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:06,235 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1038): replaying wal for 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:06,236 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:06,237 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:06,238 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1048): stopping wal replay for 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:06,238 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1060): Cleaning up temporary data for 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:06,238 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T14:17:06,240 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1093): writing seq id for 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:06,241 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1114): Opened 1a0d122b7a4f4891082438eedd4480d0; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73107978, jitterRate=0.08939376473426819}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T14:17:06,241 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:06,246 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegion(1006): Region open journal for 1a0d122b7a4f4891082438eedd4480d0: Running coprocessor pre-open hook at 1731421026211Writing region info on filesystem at 1731421026211Initializing all the Stores at 1731421026216 (+5 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731421026217 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731421026217Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731421026217Cleaning up temporary data from old regions at 1731421026238 (+21 ms)Running coprocessor post-open hooks at 1731421026241 (+3 ms)Region opened successfully at 1731421026246 (+5 ms) 2024-11-12T14:17:06,247 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., pid=33, masterSystemTime=1731421026188 2024-11-12T14:17:06,252 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:06,252 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=33}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:06,265 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=1a0d122b7a4f4891082438eedd4480d0, regionState=OPEN, openSeqNum=5, regionLocation=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:06,270 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=31, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 because future has completed 2024-11-12T14:17:06,286 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=31 2024-11-12T14:17:06,287 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=31, state=SUCCESS, hasLock=false; OpenRegionProcedure 1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 in 241 msec 2024-11-12T14:17:06,290 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-12T14:17:06,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a0d122b7a4f4891082438eedd4480d0, REOPEN/MOVE in 586 msec 2024-11-12T14:17:06,295 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-12T14:17:06,296 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 608 msec 2024-11-12T14:17:06,303 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 644 msec 2024-11-12T14:17:06,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-12T14:17:06,307 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:17:06,312 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fd04cf4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:06,312 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:17:06,313 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:17:06,319 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:17:06,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:17:06,320 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:17:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c2349e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:17:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:17:06,321 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:06,323 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55316, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:17:06,324 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@320bd837, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:06,330 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13e89b68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:06,330 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:17:06,330 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:17:06,332 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:17:06,332 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:17:06,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:17:06,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f33b51f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:06,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:17:06,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:17:06,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:06,336 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:17:06,338 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@692f67f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:06,349 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4039272, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:06,349 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:17:06,349 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:17:06,353 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:17:06,353 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:17:06,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:17:06,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b592e1f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:06,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:17:06,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:17:06,354 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:06,361 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55352, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:17:06,364 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46c07457, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:06,368 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4320bf01, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:06,369 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:17:06,369 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:17:06,370 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:17:06,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:17:06,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:17:06,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a00c27d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:06,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:17:06,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:17:06,371 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:06,373 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:17:06,374 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@627e308b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:06,378 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a179463, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:06,379 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:17:06,379 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:17:06,384 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:17:06,385 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:17:06,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:17:06,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@cea03b0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:06,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:17:06,386 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:17:06,387 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:06,391 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55412, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:17:06,392 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e53c02b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:06,397 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45887c66, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:06,397 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:17:06,397 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:17:06,401 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:17:06,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:17:06,401 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:17:06,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6217c8d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:06,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:17:06,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:17:06,402 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:06,405 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55434, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:17:06,406 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39e83198, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:06,412 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38bb1855, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:06,413 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:17:06,413 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:17:06,414 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:17:06,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:17:06,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:17:06,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67b6e200, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:06,414 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:17:06,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:17:06,415 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:06,416 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55466, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:17:06,421 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d0f42b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:06,424 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b3a2f0f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:06,424 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:17:06,424 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:17:06,427 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:17:06,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:17:06,427 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:17:06,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4829f175, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:06,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:17:06,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:17:06,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:06,435 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55486, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:17:06,437 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51798ca6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:06,444 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@626a197f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:06,445 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:17:06,445 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:17:06,446 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:17:06,446 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:17:06,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:17:06,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ea50b89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:06,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:17:06,447 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:17:06,448 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:06,449 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55498, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:17:06,451 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@20f6e5de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:06,453 DEBUG [Thread-540 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:17:06,454 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:17:06,455 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:17:06,456 DEBUG [Thread-546 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:17:06,456 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42902, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:17:06,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:17:06,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:17:06,460 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42908, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:17:06,464 DEBUG [Thread-551 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:17:06,465 DEBUG [Thread-549 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:17:06,466 DEBUG [Thread-544 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:17:06,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:17:06,467 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:17:06,469 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42910, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:17:06,471 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5] 2024-11-12T14:17:06,472 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5] 2024-11-12T14:17:06,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:17:06,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:17:06,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:17:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:17:06,473 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5] 2024-11-12T14:17:06,474 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42920, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:17:06,475 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42926, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:17:06,478 DEBUG [Thread-538 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:17:06,480 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5] 2024-11-12T14:17:06,481 DEBUG [Thread-553 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:17:06,482 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5] 2024-11-12T14:17:06,482 DEBUG [Thread-542 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:17:06,484 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:17:06,484 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:17:06,485 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:17:06,488 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:17:06,488 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:17:06,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees 2024-11-12T14:17:06,490 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42946, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:17:06,491 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42938, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:17:06,492 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5] 2024-11-12T14:17:06,494 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:17:06,495 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5] 2024-11-12T14:17:06,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-12T14:17:06,497 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=34, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:17:06,497 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:17:06,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:06,501 DEBUG [Thread-555 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:17:06,502 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:17:06,503 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:17:06,504 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a0d122b7a4f4891082438eedd4480d0 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T14:17:06,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=A 2024-11-12T14:17:06,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:06,504 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=B 2024-11-12T14:17:06,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:06,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=C 2024-11-12T14:17:06,505 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:06,508 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42962, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:17:06,510 DEBUG [Thread-553 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:17:06,510 DEBUG [Thread-553 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:17:06,513 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5] 2024-11-12T14:17:06,513 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:42974, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:17:06,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5] 2024-11-12T14:17:06,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-12T14:17:06,603 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:06,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421086583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:06,604 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:06,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421086585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:06,605 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:06,605 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:06,605 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:06,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:06,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:06,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:06,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:06,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421086588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:06,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:06,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:06,609 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:06,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:06,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421086589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:06,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:06,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:06,612 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:06,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411124168c26f70cb4398bb7268779b2f8c89_1a0d122b7a4f4891082438eedd4480d0 is 50, key is test_row_0/A:col10/1731421026500/Put/seqid=0 2024-11-12T14:17:06,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:06,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421086605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:06,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:06,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:06,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:06,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-12T14:17:06,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:06,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:06,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:06,654 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:06,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:06,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:06,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741927_1103 (size=21918) 2024-11-12T14:17:06,684 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:06,714 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:06,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421086709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:06,715 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411124168c26f70cb4398bb7268779b2f8c89_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124168c26f70cb4398bb7268779b2f8c89_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:06,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:06,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:06,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:06,716 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:06,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421086709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:06,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:06,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:06,717 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:06,728 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/92c54b383125465ab5dacc3b17b6f287, store: [table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:06,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:06,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421086720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:06,732 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:06,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421086720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:06,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:06,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:06,736 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:06,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/92c54b383125465ab5dacc3b17b6f287 is 175, key is test_row_0/A:col10/1731421026500/Put/seqid=0 2024-11-12T14:17:06,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:06,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:06,740 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:06,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:06,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421086732, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:06,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:06,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:06,753 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:06,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741928_1104 (size=65323) 2024-11-12T14:17:06,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-12T14:17:06,812 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-12T14:17:06,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:06,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:06,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:06,813 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:06,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:06,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:06,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:06,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421086924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:06,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:06,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:06,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:06,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:06,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421086935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:06,944 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:06,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:06,945 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:06,950 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:06,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421086940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:06,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:06,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:06,951 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:06,953 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T14:17:06,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:06,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421086951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:06,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:06,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:06,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:06,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421086964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:06,971 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:06,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-12T14:17:06,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:06,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:06,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:06,980 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:06,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:06,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:07,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-12T14:17:07,143 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-12T14:17:07,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:07,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:07,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:07,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:07,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:07,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:07,189 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/92c54b383125465ab5dacc3b17b6f287 2024-11-12T14:17:07,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421087240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:07,243 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:07,243 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:07,243 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:07,259 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:07,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421087252, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:07,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:07,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:07,260 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:07,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/53ed751256cf4adaa96f770f02d17b7a is 50, key is test_row_0/B:col10/1731421026500/Put/seqid=0 2024-11-12T14:17:07,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:07,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421087260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:07,265 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:07,269 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:07,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421087260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:07,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:07,284 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:07,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421087280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:07,301 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-12T14:17:07,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:07,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:07,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:07,301 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:07,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:07,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:07,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741929_1105 (size=12001) 2024-11-12T14:17:07,343 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/53ed751256cf4adaa96f770f02d17b7a 2024-11-12T14:17:07,413 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/5e34deec33b543c9bda6ad7c6384c0d9 is 50, key is test_row_0/C:col10/1731421026500/Put/seqid=0 2024-11-12T14:17:07,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741930_1106 (size=12001) 2024-11-12T14:17:07,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/5e34deec33b543c9bda6ad7c6384c0d9 2024-11-12T14:17:07,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-12T14:17:07,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:07,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:07,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:07,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] handler.RSProcedureHandler(58): pid=35 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:07,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=35 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:07,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=35 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:07,474 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/92c54b383125465ab5dacc3b17b6f287 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/92c54b383125465ab5dacc3b17b6f287 2024-11-12T14:17:07,484 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/92c54b383125465ab5dacc3b17b6f287, entries=350, sequenceid=17, filesize=63.8 K 2024-11-12T14:17:07,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/53ed751256cf4adaa96f770f02d17b7a as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/53ed751256cf4adaa96f770f02d17b7a 2024-11-12T14:17:07,498 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/53ed751256cf4adaa96f770f02d17b7a, entries=150, sequenceid=17, filesize=11.7 K 2024-11-12T14:17:07,500 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/5e34deec33b543c9bda6ad7c6384c0d9 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/5e34deec33b543c9bda6ad7c6384c0d9 2024-11-12T14:17:07,522 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/5e34deec33b543c9bda6ad7c6384c0d9, entries=150, sequenceid=17, filesize=11.7 K 2024-11-12T14:17:07,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 1a0d122b7a4f4891082438eedd4480d0 in 1026ms, sequenceid=17, compaction requested=false 2024-11-12T14:17:07,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:07,617 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=35 2024-11-12T14:17:07,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:07,617 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2902): Flushing 1a0d122b7a4f4891082438eedd4480d0 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T14:17:07,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=A 2024-11-12T14:17:07,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:07,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=B 2024-11-12T14:17:07,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:07,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=C 2024-11-12T14:17:07,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:07,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-12T14:17:07,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122d412a8196e74ffab7a4fd88899c2d98_1a0d122b7a4f4891082438eedd4480d0 is 50, key is test_row_0/A:col10/1731421026571/Put/seqid=0 2024-11-12T14:17:07,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741931_1107 (size=12154) 2024-11-12T14:17:07,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:07,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:07,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:07,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421087807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:07,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:07,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:07,824 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:07,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:07,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421087813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:07,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:07,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:07,830 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:07,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:07,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421087817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:07,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:07,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421087823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:07,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:07,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:07,832 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:07,845 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:07,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421087833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:07,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:07,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:07,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:07,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:07,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421087932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:07,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:07,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:07,938 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:07,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:07,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:07,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421087941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:07,955 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:07,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421087942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:07,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:07,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:07,958 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:07,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421087941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:07,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:07,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:07,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:07,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:07,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:07,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:07,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:07,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421087964, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:07,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:08,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:08,141 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122d412a8196e74ffab7a4fd88899c2d98_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122d412a8196e74ffab7a4fd88899c2d98_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:08,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/5a588b28cc514ca8ada4254b4d3297b6, store: [table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:08,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/5a588b28cc514ca8ada4254b4d3297b6 is 175, key is test_row_0/A:col10/1731421026571/Put/seqid=0 2024-11-12T14:17:08,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:08,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421088150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:08,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,158 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:08,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:08,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:08,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421088172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:08,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421088170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:08,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:08,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:08,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421088170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:08,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:08,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:08,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421088180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:08,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741932_1108 (size=30955) 2024-11-12T14:17:08,184 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:08,199 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/5a588b28cc514ca8ada4254b4d3297b6 2024-11-12T14:17:08,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/07dbfe027ad34b23b02b66314555a88c is 50, key is test_row_0/B:col10/1731421026571/Put/seqid=0 2024-11-12T14:17:08,249 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741933_1109 (size=12001) 2024-11-12T14:17:08,250 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/07dbfe027ad34b23b02b66314555a88c 2024-11-12T14:17:08,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/774ec1c8d09f4bed9faffca6305f61c4 is 50, key is test_row_0/C:col10/1731421026571/Put/seqid=0 2024-11-12T14:17:08,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741934_1110 (size=12001) 2024-11-12T14:17:08,316 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/774ec1c8d09f4bed9faffca6305f61c4 2024-11-12T14:17:08,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/5a588b28cc514ca8ada4254b4d3297b6 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/5a588b28cc514ca8ada4254b4d3297b6 2024-11-12T14:17:08,338 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/5a588b28cc514ca8ada4254b4d3297b6, entries=150, sequenceid=36, filesize=30.2 K 2024-11-12T14:17:08,341 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/07dbfe027ad34b23b02b66314555a88c as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/07dbfe027ad34b23b02b66314555a88c 2024-11-12T14:17:08,350 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/07dbfe027ad34b23b02b66314555a88c, entries=150, sequenceid=36, filesize=11.7 K 2024-11-12T14:17:08,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/774ec1c8d09f4bed9faffca6305f61c4 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/774ec1c8d09f4bed9faffca6305f61c4 2024-11-12T14:17:08,361 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/774ec1c8d09f4bed9faffca6305f61c4, entries=150, sequenceid=36, filesize=11.7 K 2024-11-12T14:17:08,372 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 1a0d122b7a4f4891082438eedd4480d0 in 755ms, sequenceid=36, compaction requested=false 2024-11-12T14:17:08,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.HRegion(2603): Flush status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:08,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:08,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=35}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=35 2024-11-12T14:17:08,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=35 2024-11-12T14:17:08,383 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-12T14:17:08,383 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8840 sec 2024-11-12T14:17:08,386 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=34, table=TestAcidGuarantees in 1.9000 sec 2024-11-12T14:17:08,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:08,463 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a0d122b7a4f4891082438eedd4480d0 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T14:17:08,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=A 2024-11-12T14:17:08,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:08,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=B 2024-11-12T14:17:08,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:08,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=C 2024-11-12T14:17:08,463 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:08,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411124a754f01b94b47cd9d8531023a96ea5d_1a0d122b7a4f4891082438eedd4480d0 is 50, key is test_row_0/A:col10/1731421028460/Put/seqid=0 2024-11-12T14:17:08,538 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:08,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421088519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:08,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:08,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421088519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:08,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,539 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:08,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:08,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421088531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:08,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,539 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,540 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:08,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,540 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:08,547 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:08,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421088539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:08,548 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:08,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421088544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:08,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,548 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:08,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:08,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741935_1111 (size=12154) 2024-11-12T14:17:08,564 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:08,590 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411124a754f01b94b47cd9d8531023a96ea5d_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124a754f01b94b47cd9d8531023a96ea5d_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:08,596 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/76c150bd24144407a0da98be54542466, store: [table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:08,597 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/76c150bd24144407a0da98be54542466 is 175, key is test_row_0/A:col10/1731421028460/Put/seqid=0 2024-11-12T14:17:08,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:08,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421088650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:08,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421088650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,658 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:08,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:08,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421088650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:08,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,659 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:08,660 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741936_1112 (size=30955) 2024-11-12T14:17:08,661 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:08,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-12T14:17:08,663 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:17:08,664 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:08,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421088663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:08,665 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:17:08,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:08,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421088663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:08,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:08,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees 2024-11-12T14:17:08,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,666 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:08,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T14:17:08,667 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:17:08,668 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=36, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:17:08,668 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:17:08,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T14:17:08,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T14:17:08,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:08,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:08,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:08,832 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:08,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:08,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:08,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:08,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421088870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:08,875 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:08,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,875 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,876 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:08,877 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421088871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:08,878 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,878 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,878 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:08,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:08,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421088871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:08,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,879 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421088871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,884 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:08,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421088877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:08,884 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:08,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T14:17:08,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T14:17:08,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:08,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:08,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:08,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:08,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:08,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:09,062 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=51, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/76c150bd24144407a0da98be54542466 2024-11-12T14:17:09,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/bf6f431ee86541c9b0cebf50e7045067 is 50, key is test_row_0/B:col10/1731421028460/Put/seqid=0 2024-11-12T14:17:09,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741937_1113 (size=12001) 2024-11-12T14:17:09,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/bf6f431ee86541c9b0cebf50e7045067 2024-11-12T14:17:09,134 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/fc58525cc6d545e6b24862e1771c3a8d is 50, key is test_row_0/C:col10/1731421028460/Put/seqid=0 2024-11-12T14:17:09,148 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T14:17:09,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:09,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:09,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:09,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:09,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:09,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:09,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741938_1114 (size=12001) 2024-11-12T14:17:09,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:09,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421089180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:09,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:09,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:09,191 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:09,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:09,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421089189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:09,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:09,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:09,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:09,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421089189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:09,196 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:09,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:09,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421089190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:09,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:09,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:09,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421089190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:09,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:09,197 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:09,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:09,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:09,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:09,208 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:09,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:09,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:09,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T14:17:09,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T14:17:09,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:09,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:09,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:09,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:09,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:09,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:09,464 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T14:17:09,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:09,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:09,464 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:09,465 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:09,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:09,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/fc58525cc6d545e6b24862e1771c3a8d 2024-11-12T14:17:09,589 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/76c150bd24144407a0da98be54542466 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/76c150bd24144407a0da98be54542466 2024-11-12T14:17:09,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/76c150bd24144407a0da98be54542466, entries=150, sequenceid=51, filesize=30.2 K 2024-11-12T14:17:09,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/bf6f431ee86541c9b0cebf50e7045067 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/bf6f431ee86541c9b0cebf50e7045067 2024-11-12T14:17:09,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/bf6f431ee86541c9b0cebf50e7045067, entries=150, sequenceid=51, filesize=11.7 K 2024-11-12T14:17:09,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/fc58525cc6d545e6b24862e1771c3a8d as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/fc58525cc6d545e6b24862e1771c3a8d 2024-11-12T14:17:09,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T14:17:09,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:09,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:09,629 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/fc58525cc6d545e6b24862e1771c3a8d, entries=150, sequenceid=51, filesize=11.7 K 2024-11-12T14:17:09,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:09,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:09,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:09,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for 1a0d122b7a4f4891082438eedd4480d0 in 1167ms, sequenceid=51, compaction requested=true 2024-11-12T14:17:09,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:09,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a0d122b7a4f4891082438eedd4480d0:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:17:09,631 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:09,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:09,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a0d122b7a4f4891082438eedd4480d0:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:17:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:09,631 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:09,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:09,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a0d122b7a4f4891082438eedd4480d0:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:17:09,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:09,632 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:09,632 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 1a0d122b7a4f4891082438eedd4480d0/B is initiating minor compaction (all files) 2024-11-12T14:17:09,632 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a0d122b7a4f4891082438eedd4480d0/B in TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:09,632 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/53ed751256cf4adaa96f770f02d17b7a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/07dbfe027ad34b23b02b66314555a88c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/bf6f431ee86541c9b0cebf50e7045067] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp, totalSize=35.2 K 2024-11-12T14:17:09,633 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:09,633 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 1a0d122b7a4f4891082438eedd4480d0/A is initiating minor compaction (all files) 2024-11-12T14:17:09,633 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a0d122b7a4f4891082438eedd4480d0/A in TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:09,633 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/92c54b383125465ab5dacc3b17b6f287, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/5a588b28cc514ca8ada4254b4d3297b6, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/76c150bd24144407a0da98be54542466] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp, totalSize=124.3 K 2024-11-12T14:17:09,633 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:09,634 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. files: [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/92c54b383125465ab5dacc3b17b6f287, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/5a588b28cc514ca8ada4254b4d3297b6, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/76c150bd24144407a0da98be54542466] 2024-11-12T14:17:09,637 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 92c54b383125465ab5dacc3b17b6f287, keycount=350, bloomtype=ROW, size=63.8 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731421026476 2024-11-12T14:17:09,637 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 53ed751256cf4adaa96f770f02d17b7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731421026494 2024-11-12T14:17:09,640 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a588b28cc514ca8ada4254b4d3297b6, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731421026571 2024-11-12T14:17:09,640 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 07dbfe027ad34b23b02b66314555a88c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731421026571 2024-11-12T14:17:09,641 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting bf6f431ee86541c9b0cebf50e7045067, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731421027810 2024-11-12T14:17:09,641 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 76c150bd24144407a0da98be54542466, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731421027810 2024-11-12T14:17:09,664 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:09,667 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a0d122b7a4f4891082438eedd4480d0#B#compaction#97 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:09,667 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/dbda84ef4a594ea5a9a0d51c275e71de is 50, key is test_row_0/B:col10/1731421028460/Put/seqid=0 2024-11-12T14:17:09,679 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411128f192b013bc141878a3478c0c09a8fc3_1a0d122b7a4f4891082438eedd4480d0 store=[table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:09,691 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411128f192b013bc141878a3478c0c09a8fc3_1a0d122b7a4f4891082438eedd4480d0, store=[table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:09,692 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411128f192b013bc141878a3478c0c09a8fc3_1a0d122b7a4f4891082438eedd4480d0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:09,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a0d122b7a4f4891082438eedd4480d0 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T14:17:09,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=A 2024-11-12T14:17:09,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:09,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=B 2024-11-12T14:17:09,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:09,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=C 2024-11-12T14:17:09,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:09,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:09,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741940_1116 (size=4469) 2024-11-12T14:17:09,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741939_1115 (size=12104) 2024-11-12T14:17:09,743 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411128598e75f551248179685c142f97f7f25_1a0d122b7a4f4891082438eedd4480d0 is 50, key is test_row_0/A:col10/1731421028527/Put/seqid=0 2024-11-12T14:17:09,771 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:09,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421089753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:09,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:09,772 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:09,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:09,772 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:09,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421089759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:09,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:09,784 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:09,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421089771, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:09,785 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741941_1117 (size=14594) 2024-11-12T14:17:09,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:09,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:09,785 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:09,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:09,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421089772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:09,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:09,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:09,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:09,787 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:09,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421089777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:09,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:09,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:09,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:09,791 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T14:17:09,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:09,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:09,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:09,796 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:09,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:09,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:09,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T14:17:09,813 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:09,853 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411128598e75f551248179685c142f97f7f25_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411128598e75f551248179685c142f97f7f25_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:09,855 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/82be5972d5dd44c78bad539458e5d182, store: [table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:09,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/82be5972d5dd44c78bad539458e5d182 is 175, key is test_row_0/A:col10/1731421028527/Put/seqid=0 2024-11-12T14:17:09,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:09,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421089880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:09,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:09,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:09,882 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:09,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:09,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421089884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:09,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:09,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:09,897 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:09,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:09,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421089890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:09,897 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:09,897 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:09,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:09,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421089890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:09,898 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:09,898 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:09,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:09,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:09,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421089890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:09,898 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:09,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:09,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:09,900 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:09,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741942_1118 (size=39549) 2024-11-12T14:17:09,920 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=70, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/82be5972d5dd44c78bad539458e5d182 2024-11-12T14:17:09,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T14:17:09,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:09,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:09,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:09,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:09,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:09,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:10,002 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/3710459c669b4bed84ba8bf9492e0ae5 is 50, key is test_row_0/B:col10/1731421028527/Put/seqid=0 2024-11-12T14:17:10,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741943_1119 (size=12001) 2024-11-12T14:17:10,100 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:10,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421090090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:10,101 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:10,107 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:10,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421090099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:10,109 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:10,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:10,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421090110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:10,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:10,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:10,122 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:10,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:10,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421090110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:10,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:10,123 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:10,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:10,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421090110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:10,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:10,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:10,124 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:10,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T14:17:10,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:10,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:10,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:10,132 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:10,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:10,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:10,134 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a0d122b7a4f4891082438eedd4480d0#A#compaction#96 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:10,135 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/6dce43b416b741c8b3a745e3a3b9e17e is 175, key is test_row_0/A:col10/1731421028460/Put/seqid=0 2024-11-12T14:17:10,175 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/dbda84ef4a594ea5a9a0d51c275e71de as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/dbda84ef4a594ea5a9a0d51c275e71de 2024-11-12T14:17:10,194 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a0d122b7a4f4891082438eedd4480d0/B of 1a0d122b7a4f4891082438eedd4480d0 into dbda84ef4a594ea5a9a0d51c275e71de(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:10,194 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:10,194 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., storeName=1a0d122b7a4f4891082438eedd4480d0/B, priority=13, startTime=1731421029631; duration=0sec 2024-11-12T14:17:10,194 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:10,194 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a0d122b7a4f4891082438eedd4480d0:B 2024-11-12T14:17:10,194 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:10,197 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:10,197 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 1a0d122b7a4f4891082438eedd4480d0/C is initiating minor compaction (all files) 2024-11-12T14:17:10,197 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a0d122b7a4f4891082438eedd4480d0/C in TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:10,198 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/5e34deec33b543c9bda6ad7c6384c0d9, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/774ec1c8d09f4bed9faffca6305f61c4, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/fc58525cc6d545e6b24862e1771c3a8d] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp, totalSize=35.2 K 2024-11-12T14:17:10,199 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e34deec33b543c9bda6ad7c6384c0d9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1731421026494 2024-11-12T14:17:10,199 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 774ec1c8d09f4bed9faffca6305f61c4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731421026571 2024-11-12T14:17:10,200 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting fc58525cc6d545e6b24862e1771c3a8d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731421027810 2024-11-12T14:17:10,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741944_1120 (size=31058) 2024-11-12T14:17:10,219 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/6dce43b416b741c8b3a745e3a3b9e17e as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/6dce43b416b741c8b3a745e3a3b9e17e 2024-11-12T14:17:10,229 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a0d122b7a4f4891082438eedd4480d0/A of 1a0d122b7a4f4891082438eedd4480d0 into 6dce43b416b741c8b3a745e3a3b9e17e(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:10,229 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:10,229 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., storeName=1a0d122b7a4f4891082438eedd4480d0/A, priority=13, startTime=1731421029630; duration=0sec 2024-11-12T14:17:10,229 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:10,229 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a0d122b7a4f4891082438eedd4480d0:A 2024-11-12T14:17:10,235 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a0d122b7a4f4891082438eedd4480d0#C#compaction#100 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:10,235 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/73681badb6e14a31987ebe4e137efdad is 50, key is test_row_0/C:col10/1731421028460/Put/seqid=0 2024-11-12T14:17:10,288 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T14:17:10,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:10,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:10,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:10,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:10,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:10,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:10,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741945_1121 (size=12104) 2024-11-12T14:17:10,318 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/73681badb6e14a31987ebe4e137efdad as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/73681badb6e14a31987ebe4e137efdad 2024-11-12T14:17:10,329 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a0d122b7a4f4891082438eedd4480d0/C of 1a0d122b7a4f4891082438eedd4480d0 into 73681badb6e14a31987ebe4e137efdad(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:10,329 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:10,329 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., storeName=1a0d122b7a4f4891082438eedd4480d0/C, priority=13, startTime=1731421029631; duration=0sec 2024-11-12T14:17:10,329 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:10,329 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a0d122b7a4f4891082438eedd4480d0:C 2024-11-12T14:17:10,414 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:10,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421090409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:10,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:10,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:10,415 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:10,420 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:10,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421090419, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:10,421 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:10,421 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:10,421 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:10,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:10,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421090430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:10,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:10,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:10,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:10,441 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:10,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421090431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:10,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:10,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:10,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:10,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:10,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421090432, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:10,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:10,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:10,447 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:10,450 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/3710459c669b4bed84ba8bf9492e0ae5 2024-11-12T14:17:10,453 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T14:17:10,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:10,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:10,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:10,453 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:10,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:10,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:10,494 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/dc28e453cb194ea79b125fecb4d52db6 is 50, key is test_row_0/C:col10/1731421028527/Put/seqid=0 2024-11-12T14:17:10,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741946_1122 (size=12001) 2024-11-12T14:17:10,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=70 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/dc28e453cb194ea79b125fecb4d52db6 2024-11-12T14:17:10,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/82be5972d5dd44c78bad539458e5d182 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/82be5972d5dd44c78bad539458e5d182 2024-11-12T14:17:10,618 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T14:17:10,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:10,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:10,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:10,618 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] handler.RSProcedureHandler(58): pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:10,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=37 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:10,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/82be5972d5dd44c78bad539458e5d182, entries=200, sequenceid=70, filesize=38.6 K 2024-11-12T14:17:10,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=37 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:10,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/3710459c669b4bed84ba8bf9492e0ae5 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/3710459c669b4bed84ba8bf9492e0ae5 2024-11-12T14:17:10,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/3710459c669b4bed84ba8bf9492e0ae5, entries=150, sequenceid=70, filesize=11.7 K 2024-11-12T14:17:10,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/dc28e453cb194ea79b125fecb4d52db6 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/dc28e453cb194ea79b125fecb4d52db6 2024-11-12T14:17:10,682 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/dc28e453cb194ea79b125fecb4d52db6, entries=150, sequenceid=70, filesize=11.7 K 2024-11-12T14:17:10,684 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 1a0d122b7a4f4891082438eedd4480d0 in 975ms, sequenceid=70, compaction requested=false 2024-11-12T14:17:10,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:10,777 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=37 2024-11-12T14:17:10,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:10,778 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2902): Flushing 1a0d122b7a4f4891082438eedd4480d0 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T14:17:10,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=A 2024-11-12T14:17:10,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:10,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=B 2024-11-12T14:17:10,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:10,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=C 2024-11-12T14:17:10,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:10,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112b830e3a204764affae111236b1ec6655_1a0d122b7a4f4891082438eedd4480d0 is 50, key is test_row_0/A:col10/1731421029770/Put/seqid=0 2024-11-12T14:17:10,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T14:17:10,885 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741947_1123 (size=12154) 2024-11-12T14:17:10,937 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:10,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:11,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:11,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421091014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:11,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:11,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421091016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:11,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,036 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:11,036 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:11,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421091018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,037 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:11,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,038 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:11,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:11,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421091023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:11,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:11,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421091026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:11,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,041 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:11,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:11,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:11,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421091139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:11,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:11,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421091139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:11,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:11,147 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,147 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,147 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:11,155 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:11,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:11,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421091150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:11,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:11,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421091152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:11,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:11,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:11,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421091150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:11,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:11,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:11,309 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112b830e3a204764affae111236b1ec6655_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b830e3a204764affae111236b1ec6655_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:11,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/62c20cdbdb71436fb0135f5cb6c908b3, store: [table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:11,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/62c20cdbdb71436fb0135f5cb6c908b3 is 175, key is test_row_0/A:col10/1731421029770/Put/seqid=0 2024-11-12T14:17:11,355 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:11,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421091350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:11,356 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,356 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,356 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:11,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741948_1124 (size=30955) 2024-11-12T14:17:11,360 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:11,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421091360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:11,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:11,364 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=86, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/62c20cdbdb71436fb0135f5cb6c908b3 2024-11-12T14:17:11,372 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:11,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421091372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:11,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421091372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:11,373 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:11,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421091372, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:11,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,376 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:11,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,377 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:11,377 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:11,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/0aa4a9a3203b47d9b9129cbbb08d015b is 50, key is test_row_0/B:col10/1731421029770/Put/seqid=0 2024-11-12T14:17:11,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741949_1125 (size=12001) 2024-11-12T14:17:11,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:11,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421091659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:11,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:11,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421091672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:11,682 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:11,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421091681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:11,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:11,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,685 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:11,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:11,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:11,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421091690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:11,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421091690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:11,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,691 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:11,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:11,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:11,876 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/0aa4a9a3203b47d9b9129cbbb08d015b 2024-11-12T14:17:11,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/ee31394860704199890769e87985e267 is 50, key is test_row_0/C:col10/1731421029770/Put/seqid=0 2024-11-12T14:17:12,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741950_1126 (size=12001) 2024-11-12T14:17:12,029 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=86 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/ee31394860704199890769e87985e267 2024-11-12T14:17:12,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/62c20cdbdb71436fb0135f5cb6c908b3 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/62c20cdbdb71436fb0135f5cb6c908b3 2024-11-12T14:17:12,100 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/62c20cdbdb71436fb0135f5cb6c908b3, entries=150, sequenceid=86, filesize=30.2 K 2024-11-12T14:17:12,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/0aa4a9a3203b47d9b9129cbbb08d015b as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/0aa4a9a3203b47d9b9129cbbb08d015b 2024-11-12T14:17:12,123 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/0aa4a9a3203b47d9b9129cbbb08d015b, entries=150, sequenceid=86, filesize=11.7 K 2024-11-12T14:17:12,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/ee31394860704199890769e87985e267 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/ee31394860704199890769e87985e267 2024-11-12T14:17:12,164 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/ee31394860704199890769e87985e267, entries=150, sequenceid=86, filesize=11.7 K 2024-11-12T14:17:12,169 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 1a0d122b7a4f4891082438eedd4480d0 in 1390ms, sequenceid=86, compaction requested=true 2024-11-12T14:17:12,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.HRegion(2603): Flush status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:12,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:12,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=37}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=37 2024-11-12T14:17:12,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=37 2024-11-12T14:17:12,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:12,188 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-12T14:17:12,188 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a0d122b7a4f4891082438eedd4480d0 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T14:17:12,188 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.5150 sec 2024-11-12T14:17:12,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=A 2024-11-12T14:17:12,188 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:12,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=B 2024-11-12T14:17:12,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:12,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=C 2024-11-12T14:17:12,189 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:12,202 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=36, table=TestAcidGuarantees in 3.5340 sec 2024-11-12T14:17:12,220 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112df5cb7b9bddc450aaf7409b869faeed8_1a0d122b7a4f4891082438eedd4480d0 is 50, key is test_row_0/A:col10/1731421032177/Put/seqid=0 2024-11-12T14:17:12,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:12,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421092249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:12,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741951_1127 (size=14594) 2024-11-12T14:17:12,258 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:12,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421092251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:12,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,259 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:12,260 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:12,261 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:12,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:12,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421092241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:12,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421092255, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:12,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:12,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,264 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:12,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,265 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:12,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:12,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421092262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:12,281 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,282 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:12,286 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112df5cb7b9bddc450aaf7409b869faeed8_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112df5cb7b9bddc450aaf7409b869faeed8_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:12,290 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/709143b76b084c3d9532d98b198b13e1, store: [table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:12,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/709143b76b084c3d9532d98b198b13e1 is 175, key is test_row_0/A:col10/1731421032177/Put/seqid=0 2024-11-12T14:17:12,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741952_1128 (size=39549) 2024-11-12T14:17:12,337 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=106, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/709143b76b084c3d9532d98b198b13e1 2024-11-12T14:17:12,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421092373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:12,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,389 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:12,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421092373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,390 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:12,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421092373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:12,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,392 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:12,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:12,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421092375, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:12,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/1c7723e1896047e9bda0e2d50cbf71ca is 50, key is test_row_0/B:col10/1731421032177/Put/seqid=0 2024-11-12T14:17:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,396 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:12,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421092396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:12,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,400 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:12,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741953_1129 (size=12001) 2024-11-12T14:17:12,443 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/1c7723e1896047e9bda0e2d50cbf71ca 2024-11-12T14:17:12,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/2573157dd946408a9bbab98a2aabc5dc is 50, key is test_row_0/C:col10/1731421032177/Put/seqid=0 2024-11-12T14:17:12,526 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741954_1130 (size=12001) 2024-11-12T14:17:12,533 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=106 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/2573157dd946408a9bbab98a2aabc5dc 2024-11-12T14:17:12,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/709143b76b084c3d9532d98b198b13e1 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/709143b76b084c3d9532d98b198b13e1 2024-11-12T14:17:12,621 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/709143b76b084c3d9532d98b198b13e1, entries=200, sequenceid=106, filesize=38.6 K 2024-11-12T14:17:12,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421092621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:12,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421092621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:12,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/1c7723e1896047e9bda0e2d50cbf71ca as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/1c7723e1896047e9bda0e2d50cbf71ca 2024-11-12T14:17:12,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421092621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:12,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421092621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:12,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,624 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:12,624 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:12,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,625 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:12,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,626 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:12,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/1c7723e1896047e9bda0e2d50cbf71ca, entries=150, sequenceid=106, filesize=11.7 K 2024-11-12T14:17:12,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/2573157dd946408a9bbab98a2aabc5dc as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/2573157dd946408a9bbab98a2aabc5dc 2024-11-12T14:17:12,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/2573157dd946408a9bbab98a2aabc5dc, entries=150, sequenceid=106, filesize=11.7 K 2024-11-12T14:17:12,649 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=67.09 KB/68700 for 1a0d122b7a4f4891082438eedd4480d0 in 461ms, sequenceid=106, compaction requested=true 2024-11-12T14:17:12,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:12,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a0d122b7a4f4891082438eedd4480d0:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:17:12,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:12,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a0d122b7a4f4891082438eedd4480d0:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:17:12,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T14:17:12,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a0d122b7a4f4891082438eedd4480d0:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:17:12,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-12T14:17:12,652 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:17:12,652 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:17:12,673 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:17:12,673 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 1a0d122b7a4f4891082438eedd4480d0/C is initiating minor compaction (all files) 2024-11-12T14:17:12,673 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a0d122b7a4f4891082438eedd4480d0/C in TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:12,673 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/73681badb6e14a31987ebe4e137efdad, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/dc28e453cb194ea79b125fecb4d52db6, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/ee31394860704199890769e87985e267, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/2573157dd946408a9bbab98a2aabc5dc] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp, totalSize=47.0 K 2024-11-12T14:17:12,674 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 73681badb6e14a31987ebe4e137efdad, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731421027810 2024-11-12T14:17:12,675 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting dc28e453cb194ea79b125fecb4d52db6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731421028503 2024-11-12T14:17:12,675 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting ee31394860704199890769e87985e267, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731421029724 2024-11-12T14:17:12,676 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 2573157dd946408a9bbab98a2aabc5dc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731421031013 2024-11-12T14:17:12,678 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141111 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:17:12,678 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 1a0d122b7a4f4891082438eedd4480d0/A is initiating minor compaction (all files) 2024-11-12T14:17:12,678 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a0d122b7a4f4891082438eedd4480d0/A in TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:12,678 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/6dce43b416b741c8b3a745e3a3b9e17e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/82be5972d5dd44c78bad539458e5d182, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/62c20cdbdb71436fb0135f5cb6c908b3, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/709143b76b084c3d9532d98b198b13e1] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp, totalSize=137.8 K 2024-11-12T14:17:12,679 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:12,679 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. files: [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/6dce43b416b741c8b3a745e3a3b9e17e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/82be5972d5dd44c78bad539458e5d182, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/62c20cdbdb71436fb0135f5cb6c908b3, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/709143b76b084c3d9532d98b198b13e1] 2024-11-12T14:17:12,696 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a0d122b7a4f4891082438eedd4480d0 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T14:17:12,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=A 2024-11-12T14:17:12,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:12,696 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=B 2024-11-12T14:17:12,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:12,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=C 2024-11-12T14:17:12,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:12,700 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6dce43b416b741c8b3a745e3a3b9e17e, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731421027810 2024-11-12T14:17:12,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:12,705 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82be5972d5dd44c78bad539458e5d182, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731421028503 2024-11-12T14:17:12,707 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62c20cdbdb71436fb0135f5cb6c908b3, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731421029724 2024-11-12T14:17:12,711 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a0d122b7a4f4891082438eedd4480d0#C#compaction#108 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:12,712 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/99af7a02f2c94119800665b944284cb1 is 50, key is test_row_0/C:col10/1731421032177/Put/seqid=0 2024-11-12T14:17:12,715 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 709143b76b084c3d9532d98b198b13e1, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731421031004 2024-11-12T14:17:12,751 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111203b96a7f066e4581a67e0665c9d7aefc_1a0d122b7a4f4891082438eedd4480d0 is 50, key is test_row_0/A:col10/1731421032238/Put/seqid=0 2024-11-12T14:17:12,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741955_1131 (size=12241) 2024-11-12T14:17:12,792 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:12,809 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112544935a6488342e68183b837493ae078_1a0d122b7a4f4891082438eedd4480d0 store=[table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:12,814 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/99af7a02f2c94119800665b944284cb1 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/99af7a02f2c94119800665b944284cb1 2024-11-12T14:17:12,817 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112544935a6488342e68183b837493ae078_1a0d122b7a4f4891082438eedd4480d0, store=[table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:12,818 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112544935a6488342e68183b837493ae078_1a0d122b7a4f4891082438eedd4480d0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:12,823 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1a0d122b7a4f4891082438eedd4480d0/C of 1a0d122b7a4f4891082438eedd4480d0 into 99af7a02f2c94119800665b944284cb1(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:12,823 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:12,823 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., storeName=1a0d122b7a4f4891082438eedd4480d0/C, priority=12, startTime=1731421032649; duration=0sec 2024-11-12T14:17:12,823 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:12,823 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a0d122b7a4f4891082438eedd4480d0:C 2024-11-12T14:17:12,823 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:17:12,825 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:17:12,825 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 1a0d122b7a4f4891082438eedd4480d0/B is initiating minor compaction (all files) 2024-11-12T14:17:12,825 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a0d122b7a4f4891082438eedd4480d0/B in TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:12,825 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/dbda84ef4a594ea5a9a0d51c275e71de, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/3710459c669b4bed84ba8bf9492e0ae5, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/0aa4a9a3203b47d9b9129cbbb08d015b, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/1c7723e1896047e9bda0e2d50cbf71ca] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp, totalSize=47.0 K 2024-11-12T14:17:12,825 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting dbda84ef4a594ea5a9a0d51c275e71de, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731421027810 2024-11-12T14:17:12,826 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 3710459c669b4bed84ba8bf9492e0ae5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=70, earliestPutTs=1731421028503 2024-11-12T14:17:12,829 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 0aa4a9a3203b47d9b9129cbbb08d015b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=86, earliestPutTs=1731421029724 2024-11-12T14:17:12,832 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c7723e1896047e9bda0e2d50cbf71ca, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731421031013 2024-11-12T14:17:12,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741956_1132 (size=14594) 2024-11-12T14:17:12,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=36 2024-11-12T14:17:12,839 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:17:12,840 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:17:12,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees 2024-11-12T14:17:12,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-12T14:17:12,853 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:17:12,854 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=38, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:17:12,854 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:17:12,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741957_1133 (size=4469) 2024-11-12T14:17:12,881 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a0d122b7a4f4891082438eedd4480d0#A#compaction#110 average throughput is 0.27 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:12,882 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/c225d06f910f4c8083c2fb087d65e7d5 is 175, key is test_row_0/A:col10/1731421032177/Put/seqid=0 2024-11-12T14:17:12,883 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a0d122b7a4f4891082438eedd4480d0#B#compaction#111 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:12,883 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/b0e4160f02eb42f083b1735ff28c1649 is 50, key is test_row_0/B:col10/1731421032177/Put/seqid=0 2024-11-12T14:17:12,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741958_1134 (size=31195) 2024-11-12T14:17:12,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:12,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421092932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:12,954 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:12,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421092935, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:12,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,955 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:12,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-12T14:17:12,960 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:12,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421092947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:12,960 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,961 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:12,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,964 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:12,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741959_1135 (size=12241) 2024-11-12T14:17:12,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:12,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421092954, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:12,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,970 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:12,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:12,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421092961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:12,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:12,980 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:13,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-12T14:17:13,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:13,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:13,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:13,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:13,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:13,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:13,066 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:13,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421093059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:13,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:13,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:13,067 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:13,075 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:13,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421093072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:13,077 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:13,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:13,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:13,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421093084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:13,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:13,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:13,086 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:13,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:13,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421093092, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:13,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:13,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:13,104 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:13,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-12T14:17:13,183 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-12T14:17:13,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:13,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:13,183 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:13,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:13,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:13,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:13,237 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:13,249 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111203b96a7f066e4581a67e0665c9d7aefc_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111203b96a7f066e4581a67e0665c9d7aefc_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:13,257 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/b06db45f460e41d38102f458c67550a6, store: [table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:13,257 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/b06db45f460e41d38102f458c67550a6 is 175, key is test_row_0/A:col10/1731421032238/Put/seqid=0 2024-11-12T14:17:13,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:13,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421093280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:13,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:13,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421093290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:13,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:13,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:13,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:13,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421093290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:13,292 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:13,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:13,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:13,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:13,292 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:13,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:13,293 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:13,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741960_1136 (size=39549) 2024-11-12T14:17:13,300 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/b06db45f460e41d38102f458c67550a6 2024-11-12T14:17:13,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:13,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421093309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:13,314 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:13,314 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:13,314 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:13,326 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/1b958dc5e57b4fce9078b45691122344 is 50, key is test_row_0/B:col10/1731421032238/Put/seqid=0 2024-11-12T14:17:13,349 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-12T14:17:13,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:13,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:13,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:13,349 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:13,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:13,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:13,361 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/c225d06f910f4c8083c2fb087d65e7d5 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/c225d06f910f4c8083c2fb087d65e7d5 2024-11-12T14:17:13,373 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1a0d122b7a4f4891082438eedd4480d0/A of 1a0d122b7a4f4891082438eedd4480d0 into c225d06f910f4c8083c2fb087d65e7d5(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:13,373 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:13,373 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., storeName=1a0d122b7a4f4891082438eedd4480d0/A, priority=12, startTime=1731421032649; duration=0sec 2024-11-12T14:17:13,373 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:13,373 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a0d122b7a4f4891082438eedd4480d0:A 2024-11-12T14:17:13,377 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/b0e4160f02eb42f083b1735ff28c1649 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/b0e4160f02eb42f083b1735ff28c1649 2024-11-12T14:17:13,390 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1a0d122b7a4f4891082438eedd4480d0/B of 1a0d122b7a4f4891082438eedd4480d0 into b0e4160f02eb42f083b1735ff28c1649(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:13,391 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:13,391 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., storeName=1a0d122b7a4f4891082438eedd4480d0/B, priority=12, startTime=1731421032649; duration=0sec 2024-11-12T14:17:13,391 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:13,391 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a0d122b7a4f4891082438eedd4480d0:B 2024-11-12T14:17:13,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741961_1137 (size=12001) 2024-11-12T14:17:13,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-12T14:17:13,489 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:13,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421093481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:13,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:13,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:13,489 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:13,503 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-12T14:17:13,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:13,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:13,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:13,504 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:13,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:13,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:13,605 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:13,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421093605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:13,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:13,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:13,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:13,606 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:13,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421093605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:13,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:13,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421093605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:13,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:13,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:13,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:13,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:13,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:13,608 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:13,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:13,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421093620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:13,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:13,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:13,632 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:13,660 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-12T14:17:13,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:13,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:13,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:13,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:13,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:13,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:13,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-12T14:17:13,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:13,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:13,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:13,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] handler.RSProcedureHandler(58): pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:13,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=39 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:13,824 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/1b958dc5e57b4fce9078b45691122344 2024-11-12T14:17:13,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=39 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:13,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/b449676e3a9a434bb9b2b0c9985289b9 is 50, key is test_row_0/C:col10/1731421032238/Put/seqid=0 2024-11-12T14:17:13,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741962_1138 (size=12001) 2024-11-12T14:17:13,895 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/b449676e3a9a434bb9b2b0c9985289b9 2024-11-12T14:17:13,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/b06db45f460e41d38102f458c67550a6 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/b06db45f460e41d38102f458c67550a6 2024-11-12T14:17:13,913 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/b06db45f460e41d38102f458c67550a6, entries=200, sequenceid=120, filesize=38.6 K 2024-11-12T14:17:13,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/1b958dc5e57b4fce9078b45691122344 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/1b958dc5e57b4fce9078b45691122344 2024-11-12T14:17:13,929 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/1b958dc5e57b4fce9078b45691122344, entries=150, sequenceid=120, filesize=11.7 K 2024-11-12T14:17:13,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/b449676e3a9a434bb9b2b0c9985289b9 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/b449676e3a9a434bb9b2b0c9985289b9 2024-11-12T14:17:13,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/b449676e3a9a434bb9b2b0c9985289b9, entries=150, sequenceid=120, filesize=11.7 K 2024-11-12T14:17:13,952 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 1a0d122b7a4f4891082438eedd4480d0 in 1255ms, sequenceid=120, compaction requested=false 2024-11-12T14:17:13,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:13,981 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=39 2024-11-12T14:17:13,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:13,981 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2902): Flushing 1a0d122b7a4f4891082438eedd4480d0 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T14:17:13,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=A 2024-11-12T14:17:13,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:13,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=B 2024-11-12T14:17:13,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:13,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=C 2024-11-12T14:17:13,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:14,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-12T14:17:14,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111259a9e9b596974d6dbc038837485bb602_1a0d122b7a4f4891082438eedd4480d0 is 50, key is test_row_0/A:col10/1731421032944/Put/seqid=0 2024-11-12T14:17:14,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741963_1139 (size=12254) 2024-11-12T14:17:14,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:14,115 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111259a9e9b596974d6dbc038837485bb602_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111259a9e9b596974d6dbc038837485bb602_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:14,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:14,123 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:14,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/65fc04128fab481b9c772eaf2a2574b1, store: [table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:14,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/65fc04128fab481b9c772eaf2a2574b1 is 175, key is test_row_0/A:col10/1731421032944/Put/seqid=0 2024-11-12T14:17:14,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:14,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421094153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:14,159 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,159 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:14,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,160 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:14,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421094159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:14,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,160 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:14,165 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:14,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421094159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:14,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:14,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421094161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:14,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:14,168 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:14,176 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741964_1140 (size=31055) 2024-11-12T14:17:14,180 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=142, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/65fc04128fab481b9c772eaf2a2574b1 2024-11-12T14:17:14,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/9674a4c0c6104eeba7d90ec90994cfd8 is 50, key is test_row_0/B:col10/1731421032944/Put/seqid=0 2024-11-12T14:17:14,272 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:14,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421094270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:14,273 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:14,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421094270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:14,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:14,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741965_1141 (size=12101) 2024-11-12T14:17:14,276 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:14,280 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/9674a4c0c6104eeba7d90ec90994cfd8 2024-11-12T14:17:14,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:14,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421094280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:14,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:14,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,291 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:14,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421094284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:14,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,292 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:14,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/43a28dcd73834bc58cdb7d33afbc9931 is 50, key is test_row_0/C:col10/1731421032944/Put/seqid=0 2024-11-12T14:17:14,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741966_1142 (size=12101) 2024-11-12T14:17:14,340 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/43a28dcd73834bc58cdb7d33afbc9931 2024-11-12T14:17:14,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/65fc04128fab481b9c772eaf2a2574b1 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/65fc04128fab481b9c772eaf2a2574b1 2024-11-12T14:17:14,357 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/65fc04128fab481b9c772eaf2a2574b1, entries=150, sequenceid=142, filesize=30.3 K 2024-11-12T14:17:14,358 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/9674a4c0c6104eeba7d90ec90994cfd8 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/9674a4c0c6104eeba7d90ec90994cfd8 2024-11-12T14:17:14,366 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/9674a4c0c6104eeba7d90ec90994cfd8, entries=150, sequenceid=142, filesize=11.8 K 2024-11-12T14:17:14,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/43a28dcd73834bc58cdb7d33afbc9931 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/43a28dcd73834bc58cdb7d33afbc9931 2024-11-12T14:17:14,408 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/43a28dcd73834bc58cdb7d33afbc9931, entries=150, sequenceid=142, filesize=11.8 K 2024-11-12T14:17:14,409 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for 1a0d122b7a4f4891082438eedd4480d0 in 428ms, sequenceid=142, compaction requested=true 2024-11-12T14:17:14,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.HRegion(2603): Flush status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:14,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:14,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=39}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=39 2024-11-12T14:17:14,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=39 2024-11-12T14:17:14,421 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-12T14:17:14,422 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5630 sec 2024-11-12T14:17:14,428 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=38, table=TestAcidGuarantees in 1.5850 sec 2024-11-12T14:17:14,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:14,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a0d122b7a4f4891082438eedd4480d0 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T14:17:14,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=A 2024-11-12T14:17:14,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:14,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=B 2024-11-12T14:17:14,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:14,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=C 2024-11-12T14:17:14,486 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:14,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111214919e0b1cec4087b4ea8649c0c294fe_1a0d122b7a4f4891082438eedd4480d0 is 50, key is test_row_0/A:col10/1731421034157/Put/seqid=0 2024-11-12T14:17:14,543 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:14,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421094525, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:14,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:14,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421094527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,544 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:14,544 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:14,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421094537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:14,545 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:14,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421094537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:14,545 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,545 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,546 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:14,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,548 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:14,551 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:14,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421094545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:14,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,552 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:14,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741967_1143 (size=14794) 2024-11-12T14:17:14,561 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:14,589 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111214919e0b1cec4087b4ea8649c0c294fe_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111214919e0b1cec4087b4ea8649c0c294fe_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:14,590 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/bedb384c020e4abd8be25ae325391c03, store: [table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:14,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/bedb384c020e4abd8be25ae325391c03 is 175, key is test_row_0/A:col10/1731421034157/Put/seqid=0 2024-11-12T14:17:14,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:14,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421094659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:14,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,662 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:14,662 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:14,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421094660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:14,663 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:14,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421094660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:14,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,663 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:14,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,663 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:14,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741968_1144 (size=39749) 2024-11-12T14:17:14,676 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/bedb384c020e4abd8be25ae325391c03 2024-11-12T14:17:14,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:14,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421094663, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:14,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:14,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421094664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:14,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:14,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,684 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:14,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/a7658be41bba40ae8b58fdddd4f8d30a is 50, key is test_row_0/B:col10/1731421034157/Put/seqid=0 2024-11-12T14:17:14,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741969_1145 (size=12151) 2024-11-12T14:17:14,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:14,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421094870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:14,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,872 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:14,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:14,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421094870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:14,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,872 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:14,873 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:14,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421094871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:14,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:14,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:14,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421094889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:14,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:14,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421094891, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:14,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:14,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:14,896 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:15,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=38 2024-11-12T14:17:15,020 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:17:15,021 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:17:15,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-12T14:17:15,024 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:17:15,025 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:17:15,025 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:17:15,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T14:17:15,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T14:17:15,163 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/a7658be41bba40ae8b58fdddd4f8d30a 2024-11-12T14:17:15,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T14:17:15,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:15,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:15,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:15,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:15,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:15,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:15,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421095181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:15,184 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:15,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:15,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:15,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421095182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:15,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:15,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:15,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:15,186 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:15,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:15,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:15,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421095182, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:15,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:15,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:15,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:15,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:15,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/89d7df7508d84892b5ba64a8d8a6331b is 50, key is test_row_0/C:col10/1731421034157/Put/seqid=0 2024-11-12T14:17:15,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421095200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:15,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:15,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421095200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:15,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:15,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:15,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:15,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:15,202 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:15,203 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:15,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741970_1146 (size=12151) 2024-11-12T14:17:15,343 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T14:17:15,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:15,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:15,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:15,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:15,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:15,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T14:17:15,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:15,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T14:17:15,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:15,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:15,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:15,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:15,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:15,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:15,642 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/89d7df7508d84892b5ba64a8d8a6331b 2024-11-12T14:17:15,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T14:17:15,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/bedb384c020e4abd8be25ae325391c03 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/bedb384c020e4abd8be25ae325391c03 2024-11-12T14:17:15,671 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T14:17:15,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:15,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:15,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:15,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:15,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:15,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:15,675 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/bedb384c020e4abd8be25ae325391c03, entries=200, sequenceid=158, filesize=38.8 K 2024-11-12T14:17:15,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/a7658be41bba40ae8b58fdddd4f8d30a as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/a7658be41bba40ae8b58fdddd4f8d30a 2024-11-12T14:17:15,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/a7658be41bba40ae8b58fdddd4f8d30a, entries=150, sequenceid=158, filesize=11.9 K 2024-11-12T14:17:15,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/89d7df7508d84892b5ba64a8d8a6331b as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/89d7df7508d84892b5ba64a8d8a6331b 2024-11-12T14:17:15,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:15,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421095691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:15,700 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:15,700 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:15,700 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:15,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/89d7df7508d84892b5ba64a8d8a6331b, entries=150, sequenceid=158, filesize=11.9 K 2024-11-12T14:17:15,709 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 1a0d122b7a4f4891082438eedd4480d0 in 1222ms, sequenceid=158, compaction requested=true 2024-11-12T14:17:15,709 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:15,709 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:17:15,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:15,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a0d122b7a4f4891082438eedd4480d0:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:17:15,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:15,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a0d122b7a4f4891082438eedd4480d0:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:17:15,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:15,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a0d122b7a4f4891082438eedd4480d0:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:17:15,711 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T14:17:15,711 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a0d122b7a4f4891082438eedd4480d0 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T14:17:15,711 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141548 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:17:15,711 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 1a0d122b7a4f4891082438eedd4480d0/A is initiating minor compaction (all files) 2024-11-12T14:17:15,712 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a0d122b7a4f4891082438eedd4480d0/A in TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:15,712 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/c225d06f910f4c8083c2fb087d65e7d5, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/b06db45f460e41d38102f458c67550a6, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/65fc04128fab481b9c772eaf2a2574b1, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/bedb384c020e4abd8be25ae325391c03] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp, totalSize=138.2 K 2024-11-12T14:17:15,712 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:15,712 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. files: [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/c225d06f910f4c8083c2fb087d65e7d5, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/b06db45f460e41d38102f458c67550a6, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/65fc04128fab481b9c772eaf2a2574b1, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/bedb384c020e4abd8be25ae325391c03] 2024-11-12T14:17:15,712 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:17:15,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=A 2024-11-12T14:17:15,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:15,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=B 2024-11-12T14:17:15,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:15,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=C 2024-11-12T14:17:15,713 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:15,713 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting c225d06f910f4c8083c2fb087d65e7d5, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731421031013 2024-11-12T14:17:15,714 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting b06db45f460e41d38102f458c67550a6, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731421032238 2024-11-12T14:17:15,715 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 65fc04128fab481b9c772eaf2a2574b1, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731421032778 2024-11-12T14:17:15,715 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting bedb384c020e4abd8be25ae325391c03, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731421034147 2024-11-12T14:17:15,720 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48494 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:17:15,720 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 1a0d122b7a4f4891082438eedd4480d0/B is initiating minor compaction (all files) 2024-11-12T14:17:15,720 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a0d122b7a4f4891082438eedd4480d0/B in TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:15,721 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/b0e4160f02eb42f083b1735ff28c1649, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/1b958dc5e57b4fce9078b45691122344, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/9674a4c0c6104eeba7d90ec90994cfd8, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/a7658be41bba40ae8b58fdddd4f8d30a] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp, totalSize=47.4 K 2024-11-12T14:17:15,721 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b0e4160f02eb42f083b1735ff28c1649, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731421031013 2024-11-12T14:17:15,723 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b958dc5e57b4fce9078b45691122344, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731421032238 2024-11-12T14:17:15,723 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9674a4c0c6104eeba7d90ec90994cfd8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731421032778 2024-11-12T14:17:15,724 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7658be41bba40ae8b58fdddd4f8d30a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731421034157 2024-11-12T14:17:15,743 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:15,749 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111248d31553aac04758a0aa3971210e8bf0_1a0d122b7a4f4891082438eedd4480d0 store=[table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:15,754 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111248d31553aac04758a0aa3971210e8bf0_1a0d122b7a4f4891082438eedd4480d0, store=[table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:15,755 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111248d31553aac04758a0aa3971210e8bf0_1a0d122b7a4f4891082438eedd4480d0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:15,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:15,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421095741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:15,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:15,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:15,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:15,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:15,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421095753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:15,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:15,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:15,759 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:15,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:15,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421095755, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:15,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:15,765 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:15,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:15,765 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:15,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421095757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:15,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:15,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:15,766 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:15,768 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a0d122b7a4f4891082438eedd4480d0#B#compaction#121 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:15,768 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/c0eb353038af4f9688d05723a2a2d388 is 50, key is test_row_0/B:col10/1731421034157/Put/seqid=0 2024-11-12T14:17:15,792 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d9a6a3f04a3645e89203101a4da61761_1a0d122b7a4f4891082438eedd4480d0 is 50, key is test_row_0/A:col10/1731421034533/Put/seqid=0 2024-11-12T14:17:15,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741971_1147 (size=4469) 2024-11-12T14:17:15,826 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T14:17:15,826 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:15,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:15,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:15,827 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:15,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:15,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:15,848 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741972_1148 (size=12527) 2024-11-12T14:17:15,858 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/c0eb353038af4f9688d05723a2a2d388 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/c0eb353038af4f9688d05723a2a2d388 2024-11-12T14:17:15,867 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741973_1149 (size=17284) 2024-11-12T14:17:15,872 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:15,875 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1a0d122b7a4f4891082438eedd4480d0/B of 1a0d122b7a4f4891082438eedd4480d0 into c0eb353038af4f9688d05723a2a2d388(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:15,875 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:15,875 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., storeName=1a0d122b7a4f4891082438eedd4480d0/B, priority=12, startTime=1731421035711; duration=0sec 2024-11-12T14:17:15,875 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:15,875 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a0d122b7a4f4891082438eedd4480d0:B 2024-11-12T14:17:15,875 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:17:15,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:15,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421095869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:15,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:15,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:15,876 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:15,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:15,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421095869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:15,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:15,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421095870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:15,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:15,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421095870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:15,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:15,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:15,878 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:15,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:15,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:15,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:15,880 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48494 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:17:15,880 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 1a0d122b7a4f4891082438eedd4480d0/C is initiating minor compaction (all files) 2024-11-12T14:17:15,880 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a0d122b7a4f4891082438eedd4480d0/C in TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:15,880 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/99af7a02f2c94119800665b944284cb1, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/b449676e3a9a434bb9b2b0c9985289b9, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/43a28dcd73834bc58cdb7d33afbc9931, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/89d7df7508d84892b5ba64a8d8a6331b] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp, totalSize=47.4 K 2024-11-12T14:17:15,881 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d9a6a3f04a3645e89203101a4da61761_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d9a6a3f04a3645e89203101a4da61761_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:15,882 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 99af7a02f2c94119800665b944284cb1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=106, earliestPutTs=1731421031013 2024-11-12T14:17:15,882 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/4640f9934ed049f2bcc53adc59b70c54, store: [table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:15,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/4640f9934ed049f2bcc53adc59b70c54 is 175, key is test_row_0/A:col10/1731421034533/Put/seqid=0 2024-11-12T14:17:15,883 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b449676e3a9a434bb9b2b0c9985289b9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1731421032238 2024-11-12T14:17:15,884 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43a28dcd73834bc58cdb7d33afbc9931, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731421032778 2024-11-12T14:17:15,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:15,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:15,884 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:15,885 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89d7df7508d84892b5ba64a8d8a6331b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731421034157 2024-11-12T14:17:15,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741974_1150 (size=48389) 2024-11-12T14:17:15,926 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a0d122b7a4f4891082438eedd4480d0#C#compaction#123 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:15,926 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/baa31e18942447efb8d1110804e67930 is 50, key is test_row_0/C:col10/1731421034157/Put/seqid=0 2024-11-12T14:17:15,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741975_1151 (size=12527) 2024-11-12T14:17:15,973 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/baa31e18942447efb8d1110804e67930 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/baa31e18942447efb8d1110804e67930 2024-11-12T14:17:15,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T14:17:15,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:15,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:15,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:15,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:15,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:15,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:15,997 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1a0d122b7a4f4891082438eedd4480d0/C of 1a0d122b7a4f4891082438eedd4480d0 into baa31e18942447efb8d1110804e67930(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:15,997 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:15,997 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., storeName=1a0d122b7a4f4891082438eedd4480d0/C, priority=12, startTime=1731421035711; duration=0sec 2024-11-12T14:17:15,997 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:15,997 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a0d122b7a4f4891082438eedd4480d0:C 2024-11-12T14:17:16,084 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:16,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421096082, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:16,086 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:16,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:16,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421096089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:16,091 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:16,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421096090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:16,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:16,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:16,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:16,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:16,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:16,096 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:16,096 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:16,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421096090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:16,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:16,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:16,097 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:16,141 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T14:17:16,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:16,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:16,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:16,144 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:16,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:16,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:16,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T14:17:16,232 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a0d122b7a4f4891082438eedd4480d0#A#compaction#120 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:16,233 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/86fc31a312ac498f89ad1cf6801b767b is 175, key is test_row_0/A:col10/1731421034157/Put/seqid=0 2024-11-12T14:17:16,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741976_1152 (size=31481) 2024-11-12T14:17:16,294 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/86fc31a312ac498f89ad1cf6801b767b as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/86fc31a312ac498f89ad1cf6801b767b 2024-11-12T14:17:16,314 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 1a0d122b7a4f4891082438eedd4480d0/A of 1a0d122b7a4f4891082438eedd4480d0 into 86fc31a312ac498f89ad1cf6801b767b(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:16,314 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:16,314 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., storeName=1a0d122b7a4f4891082438eedd4480d0/A, priority=12, startTime=1731421035709; duration=0sec 2024-11-12T14:17:16,314 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:16,314 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a0d122b7a4f4891082438eedd4480d0:A 2024-11-12T14:17:16,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T14:17:16,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:16,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:16,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:16,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:16,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:16,327 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=176, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/4640f9934ed049f2bcc53adc59b70c54 2024-11-12T14:17:16,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:16,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/4d7c656486da4a509f53d8c444c55dfc is 50, key is test_row_0/B:col10/1731421034533/Put/seqid=0 2024-11-12T14:17:16,368 INFO [master/2b6d221c5cde:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-12T14:17:16,368 INFO [master/2b6d221c5cde:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-12T14:17:16,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:16,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421096399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:16,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:16,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421096399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:16,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741977_1153 (size=12151) 2024-11-12T14:17:16,413 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/4d7c656486da4a509f53d8c444c55dfc 2024-11-12T14:17:16,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:16,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:16,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:16,426 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:16,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421096410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:16,426 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:16,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421096410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:16,426 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:16,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:16,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:16,427 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:16,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/1eb19deb2b85439b952349eb92cc1260 is 50, key is test_row_0/C:col10/1731421034533/Put/seqid=0 2024-11-12T14:17:16,486 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T14:17:16,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:16,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:16,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:16,487 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:16,487 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:16,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:16,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741978_1154 (size=12151) 2024-11-12T14:17:16,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=176 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/1eb19deb2b85439b952349eb92cc1260 2024-11-12T14:17:16,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/4640f9934ed049f2bcc53adc59b70c54 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/4640f9934ed049f2bcc53adc59b70c54 2024-11-12T14:17:16,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/4640f9934ed049f2bcc53adc59b70c54, entries=250, sequenceid=176, filesize=47.3 K 2024-11-12T14:17:16,624 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/4d7c656486da4a509f53d8c444c55dfc as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/4d7c656486da4a509f53d8c444c55dfc 2024-11-12T14:17:16,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T14:17:16,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:16,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:16,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:16,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:16,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:16,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:16,650 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/4d7c656486da4a509f53d8c444c55dfc, entries=150, sequenceid=176, filesize=11.9 K 2024-11-12T14:17:16,656 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/1eb19deb2b85439b952349eb92cc1260 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/1eb19deb2b85439b952349eb92cc1260 2024-11-12T14:17:16,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/1eb19deb2b85439b952349eb92cc1260, entries=150, sequenceid=176, filesize=11.9 K 2024-11-12T14:17:16,697 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 1a0d122b7a4f4891082438eedd4480d0 in 986ms, sequenceid=176, compaction requested=false 2024-11-12T14:17:16,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:16,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:16,721 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a0d122b7a4f4891082438eedd4480d0 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T14:17:16,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=A 2024-11-12T14:17:16,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:16,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=B 2024-11-12T14:17:16,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:16,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=C 2024-11-12T14:17:16,721 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:16,753 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411129efa0d70edff4501abdfc24135e57dab_1a0d122b7a4f4891082438eedd4480d0 is 50, key is test_row_0/A:col10/1731421036714/Put/seqid=0 2024-11-12T14:17:16,790 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741979_1155 (size=12304) 2024-11-12T14:17:16,799 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:16,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T14:17:16,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:16,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:16,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:16,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:16,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:16,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:16,830 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411129efa0d70edff4501abdfc24135e57dab_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129efa0d70edff4501abdfc24135e57dab_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:16,832 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/d8e516d795b840f7b93751c7469041da, store: [table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:16,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/d8e516d795b840f7b93751c7469041da is 175, key is test_row_0/A:col10/1731421036714/Put/seqid=0 2024-11-12T14:17:16,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741980_1156 (size=31105) 2024-11-12T14:17:16,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:16,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421096902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:16,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:16,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:16,911 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:16,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:16,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421096920, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:16,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:16,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:16,928 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:16,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:16,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421096930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:16,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421096932, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:16,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:16,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421096941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:16,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:16,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:16,943 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:16,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:16,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:16,945 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:16,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:16,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:16,947 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:16,961 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T14:17:16,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:16,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:16,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:16,968 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:16,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:16,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:17,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:17,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421097021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:17,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:17,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:17,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:17,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T14:17:17,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:17,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:17,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:17,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:17,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:17,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:17,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T14:17:17,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:17,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421097232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:17,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:17,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:17,239 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:17,277 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T14:17:17,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:17,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:17,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:17,277 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:17,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:17,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:17,281 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=194, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/d8e516d795b840f7b93751c7469041da 2024-11-12T14:17:17,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/e0b17b48cbdc4e91b6e80fd153a652ac is 50, key is test_row_0/B:col10/1731421036714/Put/seqid=0 2024-11-12T14:17:17,341 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741981_1157 (size=12151) 2024-11-12T14:17:17,346 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/e0b17b48cbdc4e91b6e80fd153a652ac 2024-11-12T14:17:17,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/d479e7fdd9d7408797a97d078455be65 is 50, key is test_row_0/C:col10/1731421036714/Put/seqid=0 2024-11-12T14:17:17,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741982_1158 (size=12151) 2024-11-12T14:17:17,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T14:17:17,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:17,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:17,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:17,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:17,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:17,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:17,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:17,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421097550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:17,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:17,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:17,553 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:17,592 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T14:17:17,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:17,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:17,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:17,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:17,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:17,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:17,749 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T14:17:17,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:17,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:17,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:17,751 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:17,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:17,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:17,808 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/d479e7fdd9d7408797a97d078455be65 2024-11-12T14:17:17,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/d8e516d795b840f7b93751c7469041da as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/d8e516d795b840f7b93751c7469041da 2024-11-12T14:17:17,862 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/d8e516d795b840f7b93751c7469041da, entries=150, sequenceid=194, filesize=30.4 K 2024-11-12T14:17:17,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/e0b17b48cbdc4e91b6e80fd153a652ac as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/e0b17b48cbdc4e91b6e80fd153a652ac 2024-11-12T14:17:17,897 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/e0b17b48cbdc4e91b6e80fd153a652ac, entries=150, sequenceid=194, filesize=11.9 K 2024-11-12T14:17:17,901 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/d479e7fdd9d7408797a97d078455be65 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/d479e7fdd9d7408797a97d078455be65 2024-11-12T14:17:17,909 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/d479e7fdd9d7408797a97d078455be65, entries=150, sequenceid=194, filesize=11.9 K 2024-11-12T14:17:17,909 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T14:17:17,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:17,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:17,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:17,909 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:17,909 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:17,910 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 1a0d122b7a4f4891082438eedd4480d0 in 1189ms, sequenceid=194, compaction requested=true 2024-11-12T14:17:17,910 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:17,910 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:17,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a0d122b7a4f4891082438eedd4480d0:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:17:17,912 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:17,912 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:17,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a0d122b7a4f4891082438eedd4480d0:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:17:17,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:17,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a0d122b7a4f4891082438eedd4480d0:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:17:17,913 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:17,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:17,917 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110975 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:17,917 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 1a0d122b7a4f4891082438eedd4480d0/A is initiating minor compaction (all files) 2024-11-12T14:17:17,917 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a0d122b7a4f4891082438eedd4480d0/A in TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:17,918 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/86fc31a312ac498f89ad1cf6801b767b, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/4640f9934ed049f2bcc53adc59b70c54, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/d8e516d795b840f7b93751c7469041da] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp, totalSize=108.4 K 2024-11-12T14:17:17,918 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:17,918 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. files: [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/86fc31a312ac498f89ad1cf6801b767b, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/4640f9934ed049f2bcc53adc59b70c54, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/d8e516d795b840f7b93751c7469041da] 2024-11-12T14:17:17,923 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 86fc31a312ac498f89ad1cf6801b767b, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731421034157 2024-11-12T14:17:17,924 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:17,924 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 1a0d122b7a4f4891082438eedd4480d0/B is initiating minor compaction (all files) 2024-11-12T14:17:17,924 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a0d122b7a4f4891082438eedd4480d0/B in TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:17,924 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/c0eb353038af4f9688d05723a2a2d388, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/4d7c656486da4a509f53d8c444c55dfc, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/e0b17b48cbdc4e91b6e80fd153a652ac] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp, totalSize=36.0 K 2024-11-12T14:17:17,926 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4640f9934ed049f2bcc53adc59b70c54, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731421034524 2024-11-12T14:17:17,928 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting c0eb353038af4f9688d05723a2a2d388, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731421034157 2024-11-12T14:17:17,929 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting d8e516d795b840f7b93751c7469041da, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731421035742 2024-11-12T14:17:17,933 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 4d7c656486da4a509f53d8c444c55dfc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731421034533 2024-11-12T14:17:17,936 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting e0b17b48cbdc4e91b6e80fd153a652ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731421035742 2024-11-12T14:17:17,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:17,948 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a0d122b7a4f4891082438eedd4480d0 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T14:17:17,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=A 2024-11-12T14:17:17,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:17,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=B 2024-11-12T14:17:17,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:17,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=C 2024-11-12T14:17:17,948 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:17,971 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a0d122b7a4f4891082438eedd4480d0#B#compaction#129 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:17,971 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/4692a48480b44eefbaa6feb6a62b0fac is 50, key is test_row_0/B:col10/1731421036714/Put/seqid=0 2024-11-12T14:17:17,985 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:18,006 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112f5a571fb856d4df495978057c0593937_1a0d122b7a4f4891082438eedd4480d0 is 50, key is test_row_0/A:col10/1731421036881/Put/seqid=0 2024-11-12T14:17:18,009 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112203b9ac23c1947b998ee0d0bec7d08d8_1a0d122b7a4f4891082438eedd4480d0 store=[table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:18,011 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112203b9ac23c1947b998ee0d0bec7d08d8_1a0d122b7a4f4891082438eedd4480d0, store=[table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:18,011 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112203b9ac23c1947b998ee0d0bec7d08d8_1a0d122b7a4f4891082438eedd4480d0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:18,013 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:18,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421097994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:18,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,014 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:18,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741983_1159 (size=12629) 2024-11-12T14:17:18,027 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:18,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421098012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:18,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:18,028 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:18,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421098014, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:18,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:18,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421098017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:18,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,032 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:18,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741984_1160 (size=14794) 2024-11-12T14:17:18,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,036 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:18,061 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:18,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421098061, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:18,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,062 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:18,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T14:17:18,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:18,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:18,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:18,070 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:18,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:18,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:18,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741985_1161 (size=4469) 2024-11-12T14:17:18,086 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a0d122b7a4f4891082438eedd4480d0#A#compaction#130 average throughput is 0.24 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:18,087 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/58c7fd3536754bb280af43c86d6c9cf6 is 175, key is test_row_0/A:col10/1731421036714/Put/seqid=0 2024-11-12T14:17:18,113 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741986_1162 (size=31583) 2024-11-12T14:17:18,125 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:18,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421098121, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:18,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:18,144 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:18,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421098140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,146 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:18,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:18,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:18,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421098143, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:18,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421098141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:18,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:18,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:18,172 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/58c7fd3536754bb280af43c86d6c9cf6 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/58c7fd3536754bb280af43c86d6c9cf6 2024-11-12T14:17:18,223 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a0d122b7a4f4891082438eedd4480d0/A of 1a0d122b7a4f4891082438eedd4480d0 into 58c7fd3536754bb280af43c86d6c9cf6(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:18,223 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:18,223 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., storeName=1a0d122b7a4f4891082438eedd4480d0/A, priority=13, startTime=1731421037910; duration=0sec 2024-11-12T14:17:18,223 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:18,223 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a0d122b7a4f4891082438eedd4480d0:A 2024-11-12T14:17:18,223 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:18,236 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T14:17:18,237 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:18,237 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 1a0d122b7a4f4891082438eedd4480d0/C is initiating minor compaction (all files) 2024-11-12T14:17:18,237 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a0d122b7a4f4891082438eedd4480d0/C in TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:18,237 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/baa31e18942447efb8d1110804e67930, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/1eb19deb2b85439b952349eb92cc1260, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/d479e7fdd9d7408797a97d078455be65] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp, totalSize=36.0 K 2024-11-12T14:17:18,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:18,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:18,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:18,238 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:18,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:18,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:18,241 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting baa31e18942447efb8d1110804e67930, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731421034157 2024-11-12T14:17:18,246 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1eb19deb2b85439b952349eb92cc1260, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=176, earliestPutTs=1731421034533 2024-11-12T14:17:18,252 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting d479e7fdd9d7408797a97d078455be65, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731421035742 2024-11-12T14:17:18,288 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a0d122b7a4f4891082438eedd4480d0#C#compaction#132 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:18,289 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/59d2a7624ad645eb9b7dbf335ec9e91c is 50, key is test_row_0/C:col10/1731421036714/Put/seqid=0 2024-11-12T14:17:18,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741987_1163 (size=12629) 2024-11-12T14:17:18,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:18,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421098339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:18,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:18,357 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:18,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421098350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:18,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,358 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:18,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:18,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421098360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:18,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:18,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,364 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:18,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421098360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:18,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,365 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:18,396 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/59d2a7624ad645eb9b7dbf335ec9e91c as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/59d2a7624ad645eb9b7dbf335ec9e91c 2024-11-12T14:17:18,400 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T14:17:18,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:18,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:18,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:18,400 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:18,400 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:18,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:18,423 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a0d122b7a4f4891082438eedd4480d0/C of 1a0d122b7a4f4891082438eedd4480d0 into 59d2a7624ad645eb9b7dbf335ec9e91c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:18,423 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:18,423 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., storeName=1a0d122b7a4f4891082438eedd4480d0/C, priority=13, startTime=1731421037913; duration=0sec 2024-11-12T14:17:18,423 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:18,423 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a0d122b7a4f4891082438eedd4480d0:C 2024-11-12T14:17:18,432 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:18,469 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/4692a48480b44eefbaa6feb6a62b0fac as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/4692a48480b44eefbaa6feb6a62b0fac 2024-11-12T14:17:18,480 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112f5a571fb856d4df495978057c0593937_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f5a571fb856d4df495978057c0593937_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:18,499 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/0e88359daf1a4e199e757cf8e46b99a0, store: [table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:18,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/0e88359daf1a4e199e757cf8e46b99a0 is 175, key is test_row_0/A:col10/1731421036881/Put/seqid=0 2024-11-12T14:17:18,549 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a0d122b7a4f4891082438eedd4480d0/B of 1a0d122b7a4f4891082438eedd4480d0 into 4692a48480b44eefbaa6feb6a62b0fac(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:18,549 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:18,549 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., storeName=1a0d122b7a4f4891082438eedd4480d0/B, priority=13, startTime=1731421037912; duration=0sec 2024-11-12T14:17:18,549 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:18,549 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a0d122b7a4f4891082438eedd4480d0:B 2024-11-12T14:17:18,567 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741988_1164 (size=39749) 2024-11-12T14:17:18,568 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=212, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/0e88359daf1a4e199e757cf8e46b99a0 2024-11-12T14:17:18,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T14:17:18,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:18,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:18,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:18,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:18,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:18,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:18,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/cdb67c34b56f4399b945f515b9095e89 is 50, key is test_row_0/B:col10/1731421036881/Put/seqid=0 2024-11-12T14:17:18,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741989_1165 (size=12151) 2024-11-12T14:17:18,615 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/cdb67c34b56f4399b945f515b9095e89 2024-11-12T14:17:18,634 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/0b7765b511864df2b41b82403e8179e3 is 50, key is test_row_0/C:col10/1731421036881/Put/seqid=0 2024-11-12T14:17:18,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:18,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421098659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:18,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:18,675 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741990_1166 (size=12151) 2024-11-12T14:17:18,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:18,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421098670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,678 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:18,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:18,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421098670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,679 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,680 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:18,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:18,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421098670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:18,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:18,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:18,738 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T14:17:18,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:18,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:18,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:18,738 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:18,738 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:18,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:18,891 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T14:17:18,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:18,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:18,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:18,891 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:18,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:18,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:19,054 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T14:17:19,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:19,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:19,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:19,054 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:19,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:19,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:19,077 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=212 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/0b7765b511864df2b41b82403e8179e3 2024-11-12T14:17:19,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:19,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421099080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:19,083 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_0' in 1a0d122b7a4f4891082438eedd4480d0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:17:19,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,084 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:19,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/0e88359daf1a4e199e757cf8e46b99a0 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/0e88359daf1a4e199e757cf8e46b99a0 2024-11-12T14:17:19,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/0e88359daf1a4e199e757cf8e46b99a0, entries=200, sequenceid=212, filesize=38.8 K 2024-11-12T14:17:19,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/cdb67c34b56f4399b945f515b9095e89 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/cdb67c34b56f4399b945f515b9095e89 2024-11-12T14:17:19,142 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/cdb67c34b56f4399b945f515b9095e89, entries=150, sequenceid=212, filesize=11.9 K 2024-11-12T14:17:19,143 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/0b7765b511864df2b41b82403e8179e3 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/0b7765b511864df2b41b82403e8179e3 2024-11-12T14:17:19,154 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/0b7765b511864df2b41b82403e8179e3, entries=150, sequenceid=212, filesize=11.9 K 2024-11-12T14:17:19,156 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 1a0d122b7a4f4891082438eedd4480d0 in 1207ms, sequenceid=212, compaction requested=false 2024-11-12T14:17:19,156 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:19,197 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a0d122b7a4f4891082438eedd4480d0 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T14:17:19,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:19,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=A 2024-11-12T14:17:19,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:19,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=B 2024-11-12T14:17:19,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:19,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=C 2024-11-12T14:17:19,201 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:19,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T14:17:19,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T14:17:19,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:19,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:19,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:19,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:19,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:19,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:19,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ebe063ebc57e4d9fac09de7719d4864d_1a0d122b7a4f4891082438eedd4480d0 is 50, key is test_row_0/A:col10/1731421039184/Put/seqid=0 2024-11-12T14:17:19,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741991_1167 (size=17284) 2024-11-12T14:17:19,279 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:19,285 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:19,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421099273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:19,285 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,286 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:19,289 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ebe063ebc57e4d9fac09de7719d4864d_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ebe063ebc57e4d9fac09de7719d4864d_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:19,291 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/780141815efd40c596b072b06afd2016, store: [table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:19,291 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/780141815efd40c596b072b06afd2016 is 175, key is test_row_0/A:col10/1731421039184/Put/seqid=0 2024-11-12T14:17:19,296 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:19,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421099285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:19,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:19,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:19,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421099285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:19,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:19,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:19,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421099287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:19,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,298 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:19,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741992_1168 (size=48389) 2024-11-12T14:17:19,320 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=231, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/780141815efd40c596b072b06afd2016 2024-11-12T14:17:19,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T14:17:19,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:19,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:19,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:19,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:19,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:19,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/44c81b69daf6486fb9a8e2fc261e60fb is 50, key is test_row_0/B:col10/1731421039184/Put/seqid=0 2024-11-12T14:17:19,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:19,402 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:19,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421099389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:19,403 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,403 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,403 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:19,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:19,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421099400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:19,412 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,413 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:19,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:19,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421099401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:19,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:19,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:19,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421099413, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:19,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,425 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:19,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741993_1169 (size=12151) 2024-11-12T14:17:19,439 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/44c81b69daf6486fb9a8e2fc261e60fb 2024-11-12T14:17:19,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/8528447e49b14adebdbc395d0859de94 is 50, key is test_row_0/C:col10/1731421039184/Put/seqid=0 2024-11-12T14:17:19,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741994_1170 (size=12151) 2024-11-12T14:17:19,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T14:17:19,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:19,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:19,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:19,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:19,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:19,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:19,615 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:19,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421099611, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:19,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,620 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:19,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:19,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421099620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:19,624 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:19,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421099620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:19,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,625 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:19,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:19,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:19,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421099632, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:19,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,639 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:19,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T14:17:19,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:19,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:19,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:19,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:19,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:19,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:19,847 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T14:17:19,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:19,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:19,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:19,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:19,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:19,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:19,900 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/8528447e49b14adebdbc395d0859de94 2024-11-12T14:17:19,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:19,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421099933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:19,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:19,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:19,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421099933, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:19,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:19,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:19,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421099936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:19,943 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,944 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:19,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/780141815efd40c596b072b06afd2016 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/780141815efd40c596b072b06afd2016 2024-11-12T14:17:19,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:19,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421099951, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:19,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:19,953 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:20,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/780141815efd40c596b072b06afd2016, entries=250, sequenceid=231, filesize=47.3 K 2024-11-12T14:17:20,005 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T14:17:20,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:20,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:20,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:20,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:20,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:20,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:20,014 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/44c81b69daf6486fb9a8e2fc261e60fb as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/44c81b69daf6486fb9a8e2fc261e60fb 2024-11-12T14:17:20,065 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/44c81b69daf6486fb9a8e2fc261e60fb, entries=150, sequenceid=231, filesize=11.9 K 2024-11-12T14:17:20,072 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/8528447e49b14adebdbc395d0859de94 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/8528447e49b14adebdbc395d0859de94 2024-11-12T14:17:20,093 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/8528447e49b14adebdbc395d0859de94, entries=150, sequenceid=231, filesize=11.9 K 2024-11-12T14:17:20,094 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 1a0d122b7a4f4891082438eedd4480d0 in 897ms, sequenceid=231, compaction requested=true 2024-11-12T14:17:20,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:20,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a0d122b7a4f4891082438eedd4480d0:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:17:20,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:20,094 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:20,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a0d122b7a4f4891082438eedd4480d0:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:17:20,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:20,094 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:20,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a0d122b7a4f4891082438eedd4480d0:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:17:20,094 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:20,095 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 119721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:20,095 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 1a0d122b7a4f4891082438eedd4480d0/A is initiating minor compaction (all files) 2024-11-12T14:17:20,096 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a0d122b7a4f4891082438eedd4480d0/A in TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:20,096 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/58c7fd3536754bb280af43c86d6c9cf6, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/0e88359daf1a4e199e757cf8e46b99a0, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/780141815efd40c596b072b06afd2016] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp, totalSize=116.9 K 2024-11-12T14:17:20,096 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:20,096 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. files: [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/58c7fd3536754bb280af43c86d6c9cf6, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/0e88359daf1a4e199e757cf8e46b99a0, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/780141815efd40c596b072b06afd2016] 2024-11-12T14:17:20,097 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58c7fd3536754bb280af43c86d6c9cf6, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731421035742 2024-11-12T14:17:20,097 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e88359daf1a4e199e757cf8e46b99a0, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731421036865 2024-11-12T14:17:20,097 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:20,097 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 1a0d122b7a4f4891082438eedd4480d0/B is initiating minor compaction (all files) 2024-11-12T14:17:20,097 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a0d122b7a4f4891082438eedd4480d0/B in TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:20,097 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 780141815efd40c596b072b06afd2016, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731421037981 2024-11-12T14:17:20,097 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/4692a48480b44eefbaa6feb6a62b0fac, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/cdb67c34b56f4399b945f515b9095e89, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/44c81b69daf6486fb9a8e2fc261e60fb] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp, totalSize=36.1 K 2024-11-12T14:17:20,099 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 4692a48480b44eefbaa6feb6a62b0fac, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731421035742 2024-11-12T14:17:20,099 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting cdb67c34b56f4399b945f515b9095e89, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731421036881 2024-11-12T14:17:20,101 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 44c81b69daf6486fb9a8e2fc261e60fb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731421037981 2024-11-12T14:17:20,116 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:20,122 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a0d122b7a4f4891082438eedd4480d0#B#compaction#139 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:20,123 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/235d746c25284f6a8274abff1a40f136 is 50, key is test_row_0/B:col10/1731421039184/Put/seqid=0 2024-11-12T14:17:20,124 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111232b4476009c8425fb8e3325ae308dddf_1a0d122b7a4f4891082438eedd4480d0 store=[table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:20,127 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111232b4476009c8425fb8e3325ae308dddf_1a0d122b7a4f4891082438eedd4480d0, store=[table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:20,127 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111232b4476009c8425fb8e3325ae308dddf_1a0d122b7a4f4891082438eedd4480d0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:20,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-12T14:17:20,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:20,159 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 1a0d122b7a4f4891082438eedd4480d0 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-12T14:17:20,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=A 2024-11-12T14:17:20,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:20,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=B 2024-11-12T14:17:20,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:20,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=C 2024-11-12T14:17:20,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:20,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741996_1172 (size=4469) 2024-11-12T14:17:20,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741995_1171 (size=12731) 2024-11-12T14:17:20,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112811028c80cb9475a952b647275d7ffc4_1a0d122b7a4f4891082438eedd4480d0 is 50, key is test_row_0/A:col10/1731421039247/Put/seqid=0 2024-11-12T14:17:20,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741997_1173 (size=12304) 2024-11-12T14:17:20,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:20,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:20,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:20,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421100493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:20,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:20,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:20,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:20,512 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:20,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421100499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:20,512 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:20,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:20,513 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:20,515 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:20,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421100509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:20,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:20,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:20,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:20,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:20,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421100511, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:20,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:20,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:20,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:20,571 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a0d122b7a4f4891082438eedd4480d0#A#compaction#138 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:20,572 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/3bb35026f6a644889c668521c65968d7 is 175, key is test_row_0/A:col10/1731421039184/Put/seqid=0 2024-11-12T14:17:20,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741998_1174 (size=31685) 2024-11-12T14:17:20,613 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/235d746c25284f6a8274abff1a40f136 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/235d746c25284f6a8274abff1a40f136 2024-11-12T14:17:20,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:20,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421100620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:20,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:20,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421100621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:20,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:20,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421100622, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:20,623 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:20,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:20,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:20,626 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:20,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:20,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:20,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421100630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:20,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:20,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:20,635 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:20,646 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/3bb35026f6a644889c668521c65968d7 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/3bb35026f6a644889c668521c65968d7 2024-11-12T14:17:20,647 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a0d122b7a4f4891082438eedd4480d0/B of 1a0d122b7a4f4891082438eedd4480d0 into 235d746c25284f6a8274abff1a40f136(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:20,647 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:20,647 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., storeName=1a0d122b7a4f4891082438eedd4480d0/B, priority=13, startTime=1731421040094; duration=0sec 2024-11-12T14:17:20,647 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:20,647 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a0d122b7a4f4891082438eedd4480d0:B 2024-11-12T14:17:20,647 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:20,650 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36931 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:20,650 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 1a0d122b7a4f4891082438eedd4480d0/C is initiating minor compaction (all files) 2024-11-12T14:17:20,650 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112811028c80cb9475a952b647275d7ffc4_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112811028c80cb9475a952b647275d7ffc4_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:20,650 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a0d122b7a4f4891082438eedd4480d0/C in TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:20,650 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/59d2a7624ad645eb9b7dbf335ec9e91c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/0b7765b511864df2b41b82403e8179e3, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/8528447e49b14adebdbc395d0859de94] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp, totalSize=36.1 K 2024-11-12T14:17:20,651 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 59d2a7624ad645eb9b7dbf335ec9e91c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1731421035742 2024-11-12T14:17:20,651 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b7765b511864df2b41b82403e8179e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=212, earliestPutTs=1731421036881 2024-11-12T14:17:20,651 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 8528447e49b14adebdbc395d0859de94, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731421037981 2024-11-12T14:17:20,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/472f8a1851b644d58a72b01681ea2328, store: [table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:20,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/472f8a1851b644d58a72b01681ea2328 is 175, key is test_row_0/A:col10/1731421039247/Put/seqid=0 2024-11-12T14:17:20,658 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a0d122b7a4f4891082438eedd4480d0/A of 1a0d122b7a4f4891082438eedd4480d0 into 3bb35026f6a644889c668521c65968d7(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:20,658 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:20,658 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., storeName=1a0d122b7a4f4891082438eedd4480d0/A, priority=13, startTime=1731421040094; duration=0sec 2024-11-12T14:17:20,659 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:20,659 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a0d122b7a4f4891082438eedd4480d0:A 2024-11-12T14:17:20,677 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a0d122b7a4f4891082438eedd4480d0#C#compaction#141 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:20,677 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/fb0a7f0be2be40c1bce70719a8e1192d is 50, key is test_row_0/C:col10/1731421039184/Put/seqid=0 2024-11-12T14:17:20,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741999_1175 (size=31105) 2024-11-12T14:17:20,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742000_1176 (size=12731) 2024-11-12T14:17:20,838 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:20,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421100837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:20,838 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:20,839 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:20,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:20,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421100839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:20,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:20,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421100839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:20,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:20,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:20,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:20,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:20,842 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:20,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:20,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:20,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421100844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:20,851 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:20,851 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:20,851 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:21,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:21,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421101109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:21,113 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_0' in 1a0d122b7a4f4891082438eedd4480d0 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4211 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:17:21,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:21,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:21,114 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:21,119 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=247, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/472f8a1851b644d58a72b01681ea2328 2024-11-12T14:17:21,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/b547eb2c4e924243b50681f8a30f1e73 is 50, key is test_row_0/B:col10/1731421039247/Put/seqid=0 2024-11-12T14:17:21,139 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/fb0a7f0be2be40c1bce70719a8e1192d as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/fb0a7f0be2be40c1bce70719a8e1192d 2024-11-12T14:17:21,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742001_1177 (size=12151) 2024-11-12T14:17:21,146 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a0d122b7a4f4891082438eedd4480d0/C of 1a0d122b7a4f4891082438eedd4480d0 into fb0a7f0be2be40c1bce70719a8e1192d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:21,146 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:21,147 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., storeName=1a0d122b7a4f4891082438eedd4480d0/C, priority=13, startTime=1731421040094; duration=0sec 2024-11-12T14:17:21,147 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:21,147 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a0d122b7a4f4891082438eedd4480d0:C 2024-11-12T14:17:21,150 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:21,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421101149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:21,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:21,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:21,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:21,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:21,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421101150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:21,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:21,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:21,152 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:21,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:21,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421101152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:21,156 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:21,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:21,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421101159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:21,161 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:21,161 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:21,161 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:21,545 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/b547eb2c4e924243b50681f8a30f1e73 2024-11-12T14:17:21,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/54c70a1730d147bdbc8306e939830c59 is 50, key is test_row_0/C:col10/1731421039247/Put/seqid=0 2024-11-12T14:17:21,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742002_1178 (size=12151) 2024-11-12T14:17:21,619 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=247 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/54c70a1730d147bdbc8306e939830c59 2024-11-12T14:17:21,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/472f8a1851b644d58a72b01681ea2328 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/472f8a1851b644d58a72b01681ea2328 2024-11-12T14:17:21,643 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/472f8a1851b644d58a72b01681ea2328, entries=150, sequenceid=247, filesize=30.4 K 2024-11-12T14:17:21,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/b547eb2c4e924243b50681f8a30f1e73 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/b547eb2c4e924243b50681f8a30f1e73 2024-11-12T14:17:21,663 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/b547eb2c4e924243b50681f8a30f1e73, entries=150, sequenceid=247, filesize=11.9 K 2024-11-12T14:17:21,665 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:21,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/54c70a1730d147bdbc8306e939830c59 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/54c70a1730d147bdbc8306e939830c59 2024-11-12T14:17:21,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421101661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:21,666 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:21,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:21,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:21,667 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:21,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421101661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:21,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:21,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:21,667 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:21,666 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:21,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421101661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:21,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:21,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:21,671 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:21,676 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/54c70a1730d147bdbc8306e939830c59, entries=150, sequenceid=247, filesize=11.9 K 2024-11-12T14:17:21,680 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 1a0d122b7a4f4891082438eedd4480d0 in 1520ms, sequenceid=247, compaction requested=false 2024-11-12T14:17:21,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:21,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:21,680 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-12T14:17:21,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-12T14:17:21,694 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-12T14:17:21,694 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 6.6640 sec 2024-11-12T14:17:21,696 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 6.6740 sec 2024-11-12T14:17:21,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:21,716 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a0d122b7a4f4891082438eedd4480d0 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T14:17:21,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=A 2024-11-12T14:17:21,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:21,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=B 2024-11-12T14:17:21,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:21,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=C 2024-11-12T14:17:21,717 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:21,771 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125f9c45c0fa5b42359baa9bd343de60cc_1a0d122b7a4f4891082438eedd4480d0 is 50, key is test_row_0/A:col10/1731421041691/Put/seqid=0 2024-11-12T14:17:21,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742003_1179 (size=12454) 2024-11-12T14:17:21,826 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:21,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421101823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:21,827 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:21,827 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:21,827 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:21,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:21,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421101931, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:21,933 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:21,934 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:21,934 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:22,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:22,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421102139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:22,147 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:22,147 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:22,147 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:22,197 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:22,206 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125f9c45c0fa5b42359baa9bd343de60cc_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125f9c45c0fa5b42359baa9bd343de60cc_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:22,207 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/5c5fa170a4184ea486c5855d81814a15, store: [table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:22,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/5c5fa170a4184ea486c5855d81814a15 is 175, key is test_row_0/A:col10/1731421041691/Put/seqid=0 2024-11-12T14:17:22,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742004_1180 (size=31255) 2024-11-12T14:17:22,248 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=268, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/5c5fa170a4184ea486c5855d81814a15 2024-11-12T14:17:22,267 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/78be43a73cfd48c1bf3c36bb547e04d1 is 50, key is test_row_0/B:col10/1731421041691/Put/seqid=0 2024-11-12T14:17:22,278 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742005_1181 (size=12301) 2024-11-12T14:17:22,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:22,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421102460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:22,463 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:22,464 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:22,464 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:22,680 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:22,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/78be43a73cfd48c1bf3c36bb547e04d1 2024-11-12T14:17:22,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:22,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421102680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:22,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421102679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:22,681 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_2' in 1a0d122b7a4f4891082438eedd4480d0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:17:22,681 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:22,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:22,682 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:22,682 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_2' in 1a0d122b7a4f4891082438eedd4480d0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:17:22,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:22,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:22,685 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:22,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:22,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421102691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:22,697 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_0' in 1a0d122b7a4f4891082438eedd4480d0 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2198 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:17:22,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:22,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:22,698 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:22,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/f9ab4b2a5d954a9eb6f240954434155d is 50, key is test_row_0/C:col10/1731421041691/Put/seqid=0 2024-11-12T14:17:22,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742006_1182 (size=12301) 2024-11-12T14:17:22,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/f9ab4b2a5d954a9eb6f240954434155d 2024-11-12T14:17:22,753 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/5c5fa170a4184ea486c5855d81814a15 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/5c5fa170a4184ea486c5855d81814a15 2024-11-12T14:17:22,770 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/5c5fa170a4184ea486c5855d81814a15, entries=150, sequenceid=268, filesize=30.5 K 2024-11-12T14:17:22,774 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/78be43a73cfd48c1bf3c36bb547e04d1 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/78be43a73cfd48c1bf3c36bb547e04d1 2024-11-12T14:17:22,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/78be43a73cfd48c1bf3c36bb547e04d1, entries=150, sequenceid=268, filesize=12.0 K 2024-11-12T14:17:22,785 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/f9ab4b2a5d954a9eb6f240954434155d as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/f9ab4b2a5d954a9eb6f240954434155d 2024-11-12T14:17:22,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/f9ab4b2a5d954a9eb6f240954434155d, entries=150, sequenceid=268, filesize=12.0 K 2024-11-12T14:17:22,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 1a0d122b7a4f4891082438eedd4480d0 in 1079ms, sequenceid=268, compaction requested=true 2024-11-12T14:17:22,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:22,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a0d122b7a4f4891082438eedd4480d0:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:17:22,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:22,795 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:22,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a0d122b7a4f4891082438eedd4480d0:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:17:22,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:22,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a0d122b7a4f4891082438eedd4480d0:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:17:22,795 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:22,795 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:22,802 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:22,802 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 1a0d122b7a4f4891082438eedd4480d0/B is initiating minor compaction (all files) 2024-11-12T14:17:22,802 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a0d122b7a4f4891082438eedd4480d0/B in TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:22,802 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94045 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:22,802 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/235d746c25284f6a8274abff1a40f136, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/b547eb2c4e924243b50681f8a30f1e73, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/78be43a73cfd48c1bf3c36bb547e04d1] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp, totalSize=36.3 K 2024-11-12T14:17:22,802 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 1a0d122b7a4f4891082438eedd4480d0/A is initiating minor compaction (all files) 2024-11-12T14:17:22,802 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a0d122b7a4f4891082438eedd4480d0/A in TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:22,803 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/3bb35026f6a644889c668521c65968d7, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/472f8a1851b644d58a72b01681ea2328, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/5c5fa170a4184ea486c5855d81814a15] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp, totalSize=91.8 K 2024-11-12T14:17:22,803 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:22,803 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. files: [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/3bb35026f6a644889c668521c65968d7, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/472f8a1851b644d58a72b01681ea2328, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/5c5fa170a4184ea486c5855d81814a15] 2024-11-12T14:17:22,803 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 235d746c25284f6a8274abff1a40f136, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731421037981 2024-11-12T14:17:22,803 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bb35026f6a644889c668521c65968d7, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731421037981 2024-11-12T14:17:22,804 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting b547eb2c4e924243b50681f8a30f1e73, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731421039247 2024-11-12T14:17:22,804 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 472f8a1851b644d58a72b01681ea2328, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731421039247 2024-11-12T14:17:22,804 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c5fa170a4184ea486c5855d81814a15, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731421040483 2024-11-12T14:17:22,804 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 78be43a73cfd48c1bf3c36bb547e04d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731421040483 2024-11-12T14:17:22,822 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:22,840 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112e067ac75de3a4612bb93db62d95bad0f_1a0d122b7a4f4891082438eedd4480d0 store=[table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:22,842 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a0d122b7a4f4891082438eedd4480d0#B#compaction#148 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:22,843 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112e067ac75de3a4612bb93db62d95bad0f_1a0d122b7a4f4891082438eedd4480d0, store=[table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:22,843 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e067ac75de3a4612bb93db62d95bad0f_1a0d122b7a4f4891082438eedd4480d0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:22,843 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/5560c90a10b443ae8ee2c4f2947fc91b is 50, key is test_row_0/B:col10/1731421041691/Put/seqid=0 2024-11-12T14:17:22,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742007_1183 (size=12983) 2024-11-12T14:17:22,874 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/5560c90a10b443ae8ee2c4f2947fc91b as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/5560c90a10b443ae8ee2c4f2947fc91b 2024-11-12T14:17:22,886 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a0d122b7a4f4891082438eedd4480d0/B of 1a0d122b7a4f4891082438eedd4480d0 into 5560c90a10b443ae8ee2c4f2947fc91b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:22,886 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:22,886 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., storeName=1a0d122b7a4f4891082438eedd4480d0/B, priority=13, startTime=1731421042795; duration=0sec 2024-11-12T14:17:22,886 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:22,886 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a0d122b7a4f4891082438eedd4480d0:B 2024-11-12T14:17:22,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742008_1184 (size=4469) 2024-11-12T14:17:22,886 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:22,889 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:22,890 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 1a0d122b7a4f4891082438eedd4480d0/C is initiating minor compaction (all files) 2024-11-12T14:17:22,890 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a0d122b7a4f4891082438eedd4480d0/C in TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:22,890 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/fb0a7f0be2be40c1bce70719a8e1192d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/54c70a1730d147bdbc8306e939830c59, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/f9ab4b2a5d954a9eb6f240954434155d] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp, totalSize=36.3 K 2024-11-12T14:17:22,892 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting fb0a7f0be2be40c1bce70719a8e1192d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731421037981 2024-11-12T14:17:22,893 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 54c70a1730d147bdbc8306e939830c59, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=247, earliestPutTs=1731421039247 2024-11-12T14:17:22,893 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting f9ab4b2a5d954a9eb6f240954434155d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731421040483 2024-11-12T14:17:22,931 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a0d122b7a4f4891082438eedd4480d0#C#compaction#149 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:22,931 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/6cb6ca90d3f944f58f90c6737cab93a3 is 50, key is test_row_0/C:col10/1731421041691/Put/seqid=0 2024-11-12T14:17:22,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742009_1185 (size=12983) 2024-11-12T14:17:22,971 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/6cb6ca90d3f944f58f90c6737cab93a3 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/6cb6ca90d3f944f58f90c6737cab93a3 2024-11-12T14:17:22,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:22,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a0d122b7a4f4891082438eedd4480d0 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T14:17:22,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=A 2024-11-12T14:17:22,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:22,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=B 2024-11-12T14:17:22,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:22,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=C 2024-11-12T14:17:22,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:22,995 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a0d122b7a4f4891082438eedd4480d0/C of 1a0d122b7a4f4891082438eedd4480d0 into 6cb6ca90d3f944f58f90c6737cab93a3(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:22,996 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:22,996 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., storeName=1a0d122b7a4f4891082438eedd4480d0/C, priority=13, startTime=1731421042795; duration=0sec 2024-11-12T14:17:22,996 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:22,996 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a0d122b7a4f4891082438eedd4480d0:C 2024-11-12T14:17:23,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112a9f802b91bb841708cfc063e32506037_1a0d122b7a4f4891082438eedd4480d0 is 50, key is test_row_0/A:col10/1731421041803/Put/seqid=0 2024-11-12T14:17:23,069 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742010_1186 (size=14994) 2024-11-12T14:17:23,072 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:23,079 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112a9f802b91bb841708cfc063e32506037_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a9f802b91bb841708cfc063e32506037_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:23,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:23,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421103079, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:23,082 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/083b513989794a59bdb3c16bad8dd0c6, store: [table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:23,082 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/083b513989794a59bdb3c16bad8dd0c6 is 175, key is test_row_0/A:col10/1731421041803/Put/seqid=0 2024-11-12T14:17:23,082 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:23,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:23,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:23,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742011_1187 (size=39949) 2024-11-12T14:17:23,140 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=284, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/083b513989794a59bdb3c16bad8dd0c6 2024-11-12T14:17:23,163 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/9696bad4295e408aa4e9af9f6a2a4030 is 50, key is test_row_0/B:col10/1731421041803/Put/seqid=0 2024-11-12T14:17:23,190 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:23,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421103190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:23,193 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:23,193 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:23,193 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:23,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-12T14:17:23,220 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:17:23,220 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:17:23,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742012_1188 (size=12301) 2024-11-12T14:17:23,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-12T14:17:23,223 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:17:23,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T14:17:23,225 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:17:23,225 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:17:23,286 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a0d122b7a4f4891082438eedd4480d0#A#compaction#147 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:23,287 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/f0c5e3ca377b4e13bdb081dc0e887463 is 175, key is test_row_0/A:col10/1731421041691/Put/seqid=0 2024-11-12T14:17:23,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742013_1189 (size=31937) 2024-11-12T14:17:23,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T14:17:23,345 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/f0c5e3ca377b4e13bdb081dc0e887463 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/f0c5e3ca377b4e13bdb081dc0e887463 2024-11-12T14:17:23,358 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a0d122b7a4f4891082438eedd4480d0/A of 1a0d122b7a4f4891082438eedd4480d0 into f0c5e3ca377b4e13bdb081dc0e887463(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:23,358 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:23,358 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., storeName=1a0d122b7a4f4891082438eedd4480d0/A, priority=13, startTime=1731421042795; duration=0sec 2024-11-12T14:17:23,358 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:23,358 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a0d122b7a4f4891082438eedd4480d0:A 2024-11-12T14:17:23,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T14:17:23,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:23,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:23,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:23,379 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:23,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:23,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:23,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421103399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:23,534 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T14:17:23,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:23,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:23,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:23,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:23,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:23,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:23,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T14:17:23,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/9696bad4295e408aa4e9af9f6a2a4030 2024-11-12T14:17:23,656 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/89f6ff559a014c4bb10cd9546825221e is 50, key is test_row_0/C:col10/1731421041803/Put/seqid=0 2024-11-12T14:17:23,735 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T14:17:23,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:23,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:23,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421103736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:23,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:23,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:23,737 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:23,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:23,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:23,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742014_1190 (size=12301) 2024-11-12T14:17:23,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:23,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T14:17:23,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T14:17:23,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:23,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:23,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:23,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:23,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:23,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:24,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T14:17:24,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:24,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:24,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:24,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:24,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:24,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:24,139 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/89f6ff559a014c4bb10cd9546825221e 2024-11-12T14:17:24,154 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/083b513989794a59bdb3c16bad8dd0c6 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/083b513989794a59bdb3c16bad8dd0c6 2024-11-12T14:17:24,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/083b513989794a59bdb3c16bad8dd0c6, entries=200, sequenceid=284, filesize=39.0 K 2024-11-12T14:17:24,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/9696bad4295e408aa4e9af9f6a2a4030 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/9696bad4295e408aa4e9af9f6a2a4030 2024-11-12T14:17:24,184 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/9696bad4295e408aa4e9af9f6a2a4030, entries=150, sequenceid=284, filesize=12.0 K 2024-11-12T14:17:24,186 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/89f6ff559a014c4bb10cd9546825221e as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/89f6ff559a014c4bb10cd9546825221e 2024-11-12T14:17:24,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T14:17:24,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:24,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:24,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:24,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:24,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:24,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:24,215 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/89f6ff559a014c4bb10cd9546825221e, entries=150, sequenceid=284, filesize=12.0 K 2024-11-12T14:17:24,220 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 1a0d122b7a4f4891082438eedd4480d0 in 1246ms, sequenceid=284, compaction requested=false 2024-11-12T14:17:24,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:24,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:24,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a0d122b7a4f4891082438eedd4480d0 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T14:17:24,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=A 2024-11-12T14:17:24,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:24,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=B 2024-11-12T14:17:24,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:24,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=C 2024-11-12T14:17:24,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:24,294 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112886a677a5e7944578267572ac31e045d_1a0d122b7a4f4891082438eedd4480d0 is 50, key is test_row_0/A:col10/1731421044253/Put/seqid=0 2024-11-12T14:17:24,311 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742015_1191 (size=14994) 2024-11-12T14:17:24,316 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,330 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112886a677a5e7944578267572ac31e045d_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112886a677a5e7944578267572ac31e045d_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:24,333 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/4db7f85eb73e4bd9b94de81bd2b6bc43, store: [table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:24,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/4db7f85eb73e4bd9b94de81bd2b6bc43 is 175, key is test_row_0/A:col10/1731421044253/Put/seqid=0 2024-11-12T14:17:24,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742016_1192 (size=39949) 2024-11-12T14:17:24,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T14:17:24,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T14:17:24,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:24,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:24,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:24,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:24,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:24,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:24,371 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=304, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/4db7f85eb73e4bd9b94de81bd2b6bc43 2024-11-12T14:17:24,394 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/bde243c885f54e4e8b4c0e11c29ea7c1 is 50, key is test_row_0/B:col10/1731421044253/Put/seqid=0 2024-11-12T14:17:24,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:24,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421104398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:24,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:24,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:24,415 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:24,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742017_1193 (size=12301) 2024-11-12T14:17:24,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/bde243c885f54e4e8b4c0e11c29ea7c1 2024-11-12T14:17:24,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/0ac0726e686e40c495f65e973e1f675b is 50, key is test_row_0/C:col10/1731421044253/Put/seqid=0 2024-11-12T14:17:24,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742018_1194 (size=12301) 2024-11-12T14:17:24,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=304 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/0ac0726e686e40c495f65e973e1f675b 2024-11-12T14:17:24,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/4db7f85eb73e4bd9b94de81bd2b6bc43 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/4db7f85eb73e4bd9b94de81bd2b6bc43 2024-11-12T14:17:24,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T14:17:24,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:24,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:24,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:24,534 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:24,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:24,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:24,522 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:24,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421104520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:24,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:24,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:24,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:24,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/4db7f85eb73e4bd9b94de81bd2b6bc43, entries=200, sequenceid=304, filesize=39.0 K 2024-11-12T14:17:24,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,549 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/bde243c885f54e4e8b4c0e11c29ea7c1 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/bde243c885f54e4e8b4c0e11c29ea7c1 2024-11-12T14:17:24,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/bde243c885f54e4e8b4c0e11c29ea7c1, entries=150, sequenceid=304, filesize=12.0 K 2024-11-12T14:17:24,563 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/0ac0726e686e40c495f65e973e1f675b as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/0ac0726e686e40c495f65e973e1f675b 2024-11-12T14:17:24,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,573 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/0ac0726e686e40c495f65e973e1f675b, entries=150, sequenceid=304, filesize=12.0 K 2024-11-12T14:17:24,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,574 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 1a0d122b7a4f4891082438eedd4480d0 in 314ms, sequenceid=304, compaction requested=true 2024-11-12T14:17:24,574 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,575 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:24,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a0d122b7a4f4891082438eedd4480d0:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:17:24,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:24,575 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:24,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a0d122b7a4f4891082438eedd4480d0:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:17:24,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:24,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a0d122b7a4f4891082438eedd4480d0:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:17:24,575 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:24,576 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111835 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:24,576 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 1a0d122b7a4f4891082438eedd4480d0/A is initiating minor compaction (all files) 2024-11-12T14:17:24,576 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a0d122b7a4f4891082438eedd4480d0/A in TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:24,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,576 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/f0c5e3ca377b4e13bdb081dc0e887463, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/083b513989794a59bdb3c16bad8dd0c6, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/4db7f85eb73e4bd9b94de81bd2b6bc43] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp, totalSize=109.2 K 2024-11-12T14:17:24,576 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:24,577 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. files: [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/f0c5e3ca377b4e13bdb081dc0e887463, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/083b513989794a59bdb3c16bad8dd0c6, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/4db7f85eb73e4bd9b94de81bd2b6bc43] 2024-11-12T14:17:24,577 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:24,577 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 1a0d122b7a4f4891082438eedd4480d0/B is initiating minor compaction (all files) 2024-11-12T14:17:24,577 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a0d122b7a4f4891082438eedd4480d0/B in TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:24,577 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/5560c90a10b443ae8ee2c4f2947fc91b, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/9696bad4295e408aa4e9af9f6a2a4030, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/bde243c885f54e4e8b4c0e11c29ea7c1] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp, totalSize=36.7 K 2024-11-12T14:17:24,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,578 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting f0c5e3ca377b4e13bdb081dc0e887463, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731421040483 2024-11-12T14:17:24,578 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 5560c90a10b443ae8ee2c4f2947fc91b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731421040483 2024-11-12T14:17:24,578 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 083b513989794a59bdb3c16bad8dd0c6, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731421041800 2024-11-12T14:17:24,578 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 9696bad4295e408aa4e9af9f6a2a4030, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731421041800 2024-11-12T14:17:24,579 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting bde243c885f54e4e8b4c0e11c29ea7c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731421043055 2024-11-12T14:17:24,579 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4db7f85eb73e4bd9b94de81bd2b6bc43, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731421043055 2024-11-12T14:17:24,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,616 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:24,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,623 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a0d122b7a4f4891082438eedd4480d0#B#compaction#157 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:24,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,624 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/bf3a33dd82a4462eb7e9a7a975e0794a is 50, key is test_row_0/B:col10/1731421044253/Put/seqid=0 2024-11-12T14:17:24,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,625 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112686c45482fa342cf8a0258dfe7c12d94_1a0d122b7a4f4891082438eedd4480d0 store=[table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:24,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,627 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112686c45482fa342cf8a0258dfe7c12d94_1a0d122b7a4f4891082438eedd4480d0, store=[table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:24,627 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112686c45482fa342cf8a0258dfe7c12d94_1a0d122b7a4f4891082438eedd4480d0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:24,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-12T14:17:24,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:24,693 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 1a0d122b7a4f4891082438eedd4480d0 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T14:17:24,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=A 2024-11-12T14:17:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:24,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=B 2024-11-12T14:17:24,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:24,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=C 2024-11-12T14:17:24,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:24,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,699 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742019_1195 (size=4469) 2024-11-12T14:17:24,701 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a0d122b7a4f4891082438eedd4480d0#A#compaction#156 average throughput is 0.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:24,702 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/f02301ee8ff4494eb320faf9cfa3439a is 175, key is test_row_0/A:col10/1731421044253/Put/seqid=0 2024-11-12T14:17:24,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,715 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742020_1196 (size=13085) 2024-11-12T14:17:24,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,736 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/bf3a33dd82a4462eb7e9a7a975e0794a as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/bf3a33dd82a4462eb7e9a7a975e0794a 2024-11-12T14:17:24,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,744 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a0d122b7a4f4891082438eedd4480d0/B of 1a0d122b7a4f4891082438eedd4480d0 into bf3a33dd82a4462eb7e9a7a975e0794a(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:24,744 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:24,744 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., storeName=1a0d122b7a4f4891082438eedd4480d0/B, priority=13, startTime=1731421044575; duration=0sec 2024-11-12T14:17:24,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,744 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:24,744 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a0d122b7a4f4891082438eedd4480d0:B 2024-11-12T14:17:24,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c20e8ac2fea44b8d8f0186c4bbb6568c_1a0d122b7a4f4891082438eedd4480d0 is 50, key is test_row_0/A:col10/1731421044391/Put/seqid=0 2024-11-12T14:17:24,747 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:24,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,749 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:24,749 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 1a0d122b7a4f4891082438eedd4480d0/C is initiating minor compaction (all files) 2024-11-12T14:17:24,749 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a0d122b7a4f4891082438eedd4480d0/C in TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:24,749 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/6cb6ca90d3f944f58f90c6737cab93a3, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/89f6ff559a014c4bb10cd9546825221e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/0ac0726e686e40c495f65e973e1f675b] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp, totalSize=36.7 K 2024-11-12T14:17:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,750 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 6cb6ca90d3f944f58f90c6737cab93a3, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731421040483 2024-11-12T14:17:24,751 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 89f6ff559a014c4bb10cd9546825221e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731421041800 2024-11-12T14:17:24,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742021_1197 (size=32039) 2024-11-12T14:17:24,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,753 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ac0726e686e40c495f65e973e1f675b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731421043055 2024-11-12T14:17:24,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,761 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/f02301ee8ff4494eb320faf9cfa3439a as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/f02301ee8ff4494eb320faf9cfa3439a 2024-11-12T14:17:24,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,770 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a0d122b7a4f4891082438eedd4480d0/A of 1a0d122b7a4f4891082438eedd4480d0 into f02301ee8ff4494eb320faf9cfa3439a(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:24,770 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:24,770 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., storeName=1a0d122b7a4f4891082438eedd4480d0/A, priority=13, startTime=1731421044574; duration=0sec 2024-11-12T14:17:24,771 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:24,771 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a0d122b7a4f4891082438eedd4480d0:A 2024-11-12T14:17:24,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,781 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a0d122b7a4f4891082438eedd4480d0#C#compaction#159 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:24,782 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/bb6a2e5cf4e44b949872e0eda34e8a45 is 50, key is test_row_0/C:col10/1731421044253/Put/seqid=0 2024-11-12T14:17:24,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742022_1198 (size=9914) 2024-11-12T14:17:24,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,821 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c20e8ac2fea44b8d8f0186c4bbb6568c_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c20e8ac2fea44b8d8f0186c4bbb6568c_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:24,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/904486d3b86f400794b77c8f6f6170eb, store: [table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:24,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/904486d3b86f400794b77c8f6f6170eb is 175, key is test_row_0/A:col10/1731421044391/Put/seqid=0 2024-11-12T14:17:24,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742023_1199 (size=13085) 2024-11-12T14:17:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,851 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/bb6a2e5cf4e44b949872e0eda34e8a45 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/bb6a2e5cf4e44b949872e0eda34e8a45 2024-11-12T14:17:24,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,860 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a0d122b7a4f4891082438eedd4480d0/C of 1a0d122b7a4f4891082438eedd4480d0 into bb6a2e5cf4e44b949872e0eda34e8a45(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:24,860 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:24,861 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., storeName=1a0d122b7a4f4891082438eedd4480d0/C, priority=13, startTime=1731421044575; duration=0sec 2024-11-12T14:17:24,861 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:24,861 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a0d122b7a4f4891082438eedd4480d0:C 2024-11-12T14:17:24,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742024_1200 (size=22561) 2024-11-12T14:17:24,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,878 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=318, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/904486d3b86f400794b77c8f6f6170eb 2024-11-12T14:17:24,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:24,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:24,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. as already flushing 2024-11-12T14:17:24,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/e61aa820d3fe4d5d837840262c0305ef is 50, key is test_row_0/B:col10/1731421044391/Put/seqid=0 2024-11-12T14:17:24,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:24,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421104930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:24,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:24,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421104934, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:24,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:24,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:24,949 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:24,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:24,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:24,950 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:24,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:24,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421104949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:24,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:24,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421104950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:24,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:24,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:24,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:24,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:24,963 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:24,963 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:24,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742025_1201 (size=9857) 2024-11-12T14:17:24,980 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/e61aa820d3fe4d5d837840262c0305ef 2024-11-12T14:17:25,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/a5205d7be1d9404590c035499ee1165d is 50, key is test_row_0/C:col10/1731421044391/Put/seqid=0 2024-11-12T14:17:25,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742026_1202 (size=9857) 2024-11-12T14:17:25,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:25,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421105060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:25,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:25,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421105060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:25,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,073 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:25,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,074 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:25,077 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:25,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421105069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:25,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,078 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:25,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:25,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421105069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:25,079 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,080 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:25,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:25,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42920 deadline: 1731421105133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:25,145 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_0' in 1a0d122b7a4f4891082438eedd4480d0 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:17:25,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:25,281 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:25,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421105280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:25,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:25,283 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:25,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421105280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:25,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,284 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:25,297 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:25,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421105293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:25,298 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,298 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,298 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:25,298 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:25,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421105294, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:25,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,302 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:25,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T14:17:25,458 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=318 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/a5205d7be1d9404590c035499ee1165d 2024-11-12T14:17:25,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/904486d3b86f400794b77c8f6f6170eb as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/904486d3b86f400794b77c8f6f6170eb 2024-11-12T14:17:25,475 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/904486d3b86f400794b77c8f6f6170eb, entries=100, sequenceid=318, filesize=22.0 K 2024-11-12T14:17:25,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/e61aa820d3fe4d5d837840262c0305ef as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/e61aa820d3fe4d5d837840262c0305ef 2024-11-12T14:17:25,489 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/e61aa820d3fe4d5d837840262c0305ef, entries=100, sequenceid=318, filesize=9.6 K 2024-11-12T14:17:25,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/a5205d7be1d9404590c035499ee1165d as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/a5205d7be1d9404590c035499ee1165d 2024-11-12T14:17:25,499 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/a5205d7be1d9404590c035499ee1165d, entries=100, sequenceid=318, filesize=9.6 K 2024-11-12T14:17:25,501 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for 1a0d122b7a4f4891082438eedd4480d0 in 808ms, sequenceid=318, compaction requested=false 2024-11-12T14:17:25,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:25,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:25,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-12T14:17:25,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-12T14:17:25,506 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-12T14:17:25,506 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2780 sec 2024-11-12T14:17:25,508 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 2.2860 sec 2024-11-12T14:17:25,610 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a0d122b7a4f4891082438eedd4480d0 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-12T14:17:25,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=A 2024-11-12T14:17:25,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:17:25,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=B 2024-11-12T14:17:25,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:17:25,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=C 2024-11-12T14:17:25,611 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:17:25,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:25,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411128075adfb52e04115b99404267f907ce5_1a0d122b7a4f4891082438eedd4480d0 is 50, key is test_row_0/A:col10/1731421045595/Put/seqid=0 2024-11-12T14:17:25,660 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:25,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421105652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:25,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:25,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:25,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421105655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:25,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,673 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:25,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:25,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421105659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:25,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,691 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,692 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:25,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742027_1203 (size=17534) 2024-11-12T14:17:25,708 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:25,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421105659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:25,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,709 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:25,777 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:25,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421105769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,781 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:25,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:25,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421105780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:25,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:25,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:25,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421105800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:25,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,813 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:25,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:25,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421105820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:25,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:25,823 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:25,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:25,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421105989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:26,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:26,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:26,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:26,028 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:26,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421106020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:26,030 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:26,030 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:26,030 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:26,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:26,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421106029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:26,035 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:26,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421106030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:26,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:26,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:26,037 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:26,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:26,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:26,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:26,100 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:26,123 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411128075adfb52e04115b99404267f907ce5_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411128075adfb52e04115b99404267f907ce5_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:26,125 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/3c2e6c0463e04f08b4783df2570e3eba, store: [table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:26,126 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/3c2e6c0463e04f08b4783df2570e3eba is 175, key is test_row_0/A:col10/1731421045595/Put/seqid=0 2024-11-12T14:17:26,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742028_1204 (size=48639) 2024-11-12T14:17:26,176 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=342, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/3c2e6c0463e04f08b4783df2570e3eba 2024-11-12T14:17:26,196 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/1c12884922c4422faeefb391270f13e6 is 50, key is test_row_0/B:col10/1731421045595/Put/seqid=0 2024-11-12T14:17:26,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742029_1205 (size=12301) 2024-11-12T14:17:26,235 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/1c12884922c4422faeefb391270f13e6 2024-11-12T14:17:26,262 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/60d06e821f8f489b8d2c3444d5ffd6ce is 50, key is test_row_0/C:col10/1731421045595/Put/seqid=0 2024-11-12T14:17:26,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742030_1206 (size=12301) 2024-11-12T14:17:26,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/60d06e821f8f489b8d2c3444d5ffd6ce 2024-11-12T14:17:26,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/3c2e6c0463e04f08b4783df2570e3eba as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/3c2e6c0463e04f08b4783df2570e3eba 2024-11-12T14:17:26,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:26,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42938 deadline: 1731421106309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:26,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:26,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:26,314 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:26,315 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/3c2e6c0463e04f08b4783df2570e3eba, entries=250, sequenceid=342, filesize=47.5 K 2024-11-12T14:17:26,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/1c12884922c4422faeefb391270f13e6 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/1c12884922c4422faeefb391270f13e6 2024-11-12T14:17:26,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/1c12884922c4422faeefb391270f13e6, entries=150, sequenceid=342, filesize=12.0 K 2024-11-12T14:17:26,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/60d06e821f8f489b8d2c3444d5ffd6ce as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/60d06e821f8f489b8d2c3444d5ffd6ce 2024-11-12T14:17:26,343 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:26,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42902 deadline: 1731421106339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:26,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:26,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:26,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:26,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:26,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42908 deadline: 1731421106349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:26,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:26,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:26,354 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:26,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:26,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:42946 deadline: 1731421106352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:26,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:26,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:26,357 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:17:26,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/60d06e821f8f489b8d2c3444d5ffd6ce, entries=150, sequenceid=342, filesize=12.0 K 2024-11-12T14:17:26,360 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 1a0d122b7a4f4891082438eedd4480d0 in 749ms, sequenceid=342, compaction requested=true 2024-11-12T14:17:26,360 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:26,360 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:26,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a0d122b7a4f4891082438eedd4480d0:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:17:26,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:26,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a0d122b7a4f4891082438eedd4480d0:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:17:26,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:26,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 1a0d122b7a4f4891082438eedd4480d0:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:17:26,360 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T14:17:26,361 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 103239 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:26,361 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 1a0d122b7a4f4891082438eedd4480d0/A is initiating minor compaction (all files) 2024-11-12T14:17:26,361 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a0d122b7a4f4891082438eedd4480d0/A in TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:26,362 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/f02301ee8ff4494eb320faf9cfa3439a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/904486d3b86f400794b77c8f6f6170eb, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/3c2e6c0463e04f08b4783df2570e3eba] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp, totalSize=100.8 K 2024-11-12T14:17:26,362 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:26,362 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. files: [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/f02301ee8ff4494eb320faf9cfa3439a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/904486d3b86f400794b77c8f6f6170eb, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/3c2e6c0463e04f08b4783df2570e3eba] 2024-11-12T14:17:26,362 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting f02301ee8ff4494eb320faf9cfa3439a, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731421043055 2024-11-12T14:17:26,363 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 904486d3b86f400794b77c8f6f6170eb, keycount=100, bloomtype=ROW, size=22.0 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731421044337 2024-11-12T14:17:26,364 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c2e6c0463e04f08b4783df2570e3eba, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731421044928 2024-11-12T14:17:26,364 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:26,366 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:26,366 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 1a0d122b7a4f4891082438eedd4480d0/B is initiating minor compaction (all files) 2024-11-12T14:17:26,366 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a0d122b7a4f4891082438eedd4480d0/B in TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:26,366 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/bf3a33dd82a4462eb7e9a7a975e0794a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/e61aa820d3fe4d5d837840262c0305ef, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/1c12884922c4422faeefb391270f13e6] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp, totalSize=34.4 K 2024-11-12T14:17:26,366 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf3a33dd82a4462eb7e9a7a975e0794a, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731421043055 2024-11-12T14:17:26,368 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting e61aa820d3fe4d5d837840262c0305ef, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731421044337 2024-11-12T14:17:26,368 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c12884922c4422faeefb391270f13e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731421044931 2024-11-12T14:17:26,371 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:26,383 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a0d122b7a4f4891082438eedd4480d0#B#compaction#166 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:26,383 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/18b41272526641f488a48c048cdfcb23 is 50, key is test_row_0/B:col10/1731421045595/Put/seqid=0 2024-11-12T14:17:26,385 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111298e8735575654c599f5b7ec526c912c4_1a0d122b7a4f4891082438eedd4480d0 store=[table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:26,387 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111298e8735575654c599f5b7ec526c912c4_1a0d122b7a4f4891082438eedd4480d0, store=[table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:26,387 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111298e8735575654c599f5b7ec526c912c4_1a0d122b7a4f4891082438eedd4480d0 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:26,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742031_1207 (size=13187) 2024-11-12T14:17:26,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742032_1208 (size=4469) 2024-11-12T14:17:26,425 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/18b41272526641f488a48c048cdfcb23 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/18b41272526641f488a48c048cdfcb23 2024-11-12T14:17:26,435 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a0d122b7a4f4891082438eedd4480d0/B of 1a0d122b7a4f4891082438eedd4480d0 into 18b41272526641f488a48c048cdfcb23(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:26,435 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:26,435 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., storeName=1a0d122b7a4f4891082438eedd4480d0/B, priority=13, startTime=1731421046360; duration=0sec 2024-11-12T14:17:26,436 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:26,436 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a0d122b7a4f4891082438eedd4480d0:B 2024-11-12T14:17:26,436 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:26,437 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35243 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:26,437 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 1a0d122b7a4f4891082438eedd4480d0/C is initiating minor compaction (all files) 2024-11-12T14:17:26,437 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 1a0d122b7a4f4891082438eedd4480d0/C in TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:26,438 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/bb6a2e5cf4e44b949872e0eda34e8a45, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/a5205d7be1d9404590c035499ee1165d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/60d06e821f8f489b8d2c3444d5ffd6ce] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp, totalSize=34.4 K 2024-11-12T14:17:26,439 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb6a2e5cf4e44b949872e0eda34e8a45, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=304, earliestPutTs=1731421043055 2024-11-12T14:17:26,440 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5205d7be1d9404590c035499ee1165d, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=318, earliestPutTs=1731421044337 2024-11-12T14:17:26,440 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 60d06e821f8f489b8d2c3444d5ffd6ce, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=342, earliestPutTs=1731421044931 2024-11-12T14:17:26,461 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a0d122b7a4f4891082438eedd4480d0#C#compaction#167 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:26,462 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/2e7397c2346e4c97851b1501fda220d7 is 50, key is test_row_0/C:col10/1731421045595/Put/seqid=0 2024-11-12T14:17:26,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742033_1209 (size=13187) 2024-11-12T14:17:26,505 INFO [Thread-549 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-549. 2024-11-12T14:17:26,505 DEBUG [Thread-549 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:17:26,505 DEBUG [Thread-549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:26,505 DEBUG [Thread-549 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:26,506 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:17:26,507 INFO [Thread-555 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-555. 2024-11-12T14:17:26,507 DEBUG [Thread-555 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:17:26,507 DEBUG [Thread-555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:26,507 DEBUG [Thread-555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:26,508 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:17:26,509 INFO [Thread-551 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-551. 2024-11-12T14:17:26,509 DEBUG [Thread-551 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:17:26,509 DEBUG [Thread-551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:26,509 DEBUG [Thread-551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:26,509 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:17:26,510 INFO [Thread-553 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-553. 2024-11-12T14:17:26,510 DEBUG [Thread-553 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:17:26,510 DEBUG [Thread-553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:26,510 DEBUG [Thread-553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:26,523 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:17:26,537 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/2e7397c2346e4c97851b1501fda220d7 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/2e7397c2346e4c97851b1501fda220d7 2024-11-12T14:17:26,544 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a0d122b7a4f4891082438eedd4480d0/C of 1a0d122b7a4f4891082438eedd4480d0 into 2e7397c2346e4c97851b1501fda220d7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:26,544 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:26,544 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., storeName=1a0d122b7a4f4891082438eedd4480d0/C, priority=13, startTime=1731421046360; duration=0sec 2024-11-12T14:17:26,544 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:26,544 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a0d122b7a4f4891082438eedd4480d0:C 2024-11-12T14:17:26,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:26,821 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 1a0d122b7a4f4891082438eedd4480d0 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T14:17:26,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=A 2024-11-12T14:17:26,821 INFO [Thread-542 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-542. 2024-11-12T14:17:26,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:26,821 DEBUG [Thread-542 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:17:26,821 DEBUG [Thread-542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:26,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=B 2024-11-12T14:17:26,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:26,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=C 2024-11-12T14:17:26,821 DEBUG [Thread-542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:26,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:26,822 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:17:26,830 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 1a0d122b7a4f4891082438eedd4480d0#A#compaction#165 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:26,831 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/afeeefee6fef40a9b9437612e99e85a3 is 175, key is test_row_0/A:col10/1731421045595/Put/seqid=0 2024-11-12T14:17:26,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111285669a7aee6e48299460570616b4d9dd_1a0d122b7a4f4891082438eedd4480d0 is 50, key is test_row_0/A:col10/1731421045627/Put/seqid=0 2024-11-12T14:17:26,862 INFO [Thread-546 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-546. 2024-11-12T14:17:26,862 INFO [Thread-540 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-540. 2024-11-12T14:17:26,862 DEBUG [Thread-546 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:17:26,862 DEBUG [Thread-540 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:17:26,862 DEBUG [Thread-546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:26,862 DEBUG [Thread-540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:26,862 DEBUG [Thread-540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:26,862 DEBUG [Thread-546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:26,862 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:17:26,862 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:17:26,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742034_1210 (size=32141) 2024-11-12T14:17:26,873 INFO [Thread-538 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-538. 2024-11-12T14:17:26,873 DEBUG [Thread-538 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:17:26,873 DEBUG [Thread-538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:26,873 DEBUG [Thread-538 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:26,873 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:17:26,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742035_1211 (size=12454) 2024-11-12T14:17:26,889 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:26,904 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111285669a7aee6e48299460570616b4d9dd_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111285669a7aee6e48299460570616b4d9dd_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:26,906 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/abdff07dc4904b8f834b4525f829fc27, store: [table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:26,906 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/abdff07dc4904b8f834b4525f829fc27 is 175, key is test_row_0/A:col10/1731421045627/Put/seqid=0 2024-11-12T14:17:26,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742036_1212 (size=31255) 2024-11-12T14:17:27,275 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/afeeefee6fef40a9b9437612e99e85a3 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/afeeefee6fef40a9b9437612e99e85a3 2024-11-12T14:17:27,280 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 1a0d122b7a4f4891082438eedd4480d0/A of 1a0d122b7a4f4891082438eedd4480d0 into afeeefee6fef40a9b9437612e99e85a3(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:27,280 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:27,280 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., storeName=1a0d122b7a4f4891082438eedd4480d0/A, priority=13, startTime=1731421046360; duration=0sec 2024-11-12T14:17:27,280 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:27,280 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 1a0d122b7a4f4891082438eedd4480d0:A 2024-11-12T14:17:27,360 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=356, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/abdff07dc4904b8f834b4525f829fc27 2024-11-12T14:17:27,369 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/2b70b9b6e4564ed585e3b23506cb8639 is 50, key is test_row_0/B:col10/1731421045627/Put/seqid=0 2024-11-12T14:17:27,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742037_1213 (size=12301) 2024-11-12T14:17:27,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-12T14:17:27,379 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:17:27,778 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/2b70b9b6e4564ed585e3b23506cb8639 2024-11-12T14:17:27,790 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/807d2354f5ff4caca889f1d809ab0d80 is 50, key is test_row_0/C:col10/1731421045627/Put/seqid=0 2024-11-12T14:17:27,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742038_1214 (size=12301) 2024-11-12T14:17:28,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/807d2354f5ff4caca889f1d809ab0d80 2024-11-12T14:17:28,247 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/abdff07dc4904b8f834b4525f829fc27 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/abdff07dc4904b8f834b4525f829fc27 2024-11-12T14:17:28,259 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/abdff07dc4904b8f834b4525f829fc27, entries=150, sequenceid=356, filesize=30.5 K 2024-11-12T14:17:28,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/2b70b9b6e4564ed585e3b23506cb8639 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/2b70b9b6e4564ed585e3b23506cb8639 2024-11-12T14:17:28,268 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/2b70b9b6e4564ed585e3b23506cb8639, entries=150, sequenceid=356, filesize=12.0 K 2024-11-12T14:17:28,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/807d2354f5ff4caca889f1d809ab0d80 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/807d2354f5ff4caca889f1d809ab0d80 2024-11-12T14:17:28,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/807d2354f5ff4caca889f1d809ab0d80, entries=150, sequenceid=356, filesize=12.0 K 2024-11-12T14:17:28,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=20.13 KB/20610 for 1a0d122b7a4f4891082438eedd4480d0 in 1461ms, sequenceid=356, compaction requested=false 2024-11-12T14:17:28,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 1a0d122b7a4f4891082438eedd4480d0: 2024-11-12T14:17:28,920 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T14:17:35,257 INFO [Thread-544 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-544. 2024-11-12T14:17:35,257 DEBUG [Thread-544 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:17:35,257 DEBUG [Thread-544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:35,257 DEBUG [Thread-544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:35,257 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:17:35,260 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T14:17:35,260 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 50 2024-11-12T14:17:35,260 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-12T14:17:35,260 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 56 2024-11-12T14:17:35,260 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-12T14:17:35,260 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-12T14:17:35,260 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T14:17:35,260 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1593 2024-11-12T14:17:35,260 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1519 2024-11-12T14:17:35,260 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T14:17:35,260 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 726 2024-11-12T14:17:35,260 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2178 rows 2024-11-12T14:17:35,260 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 724 2024-11-12T14:17:35,260 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2172 rows 2024-11-12T14:17:35,260 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T14:17:35,261 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T14:17:35,261 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:35,261 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:35,261 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:17:35,262 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-12T14:17:35,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T14:17:35,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T14:17:35,266 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731421055265"}]},"ts":"1731421055265"} 2024-11-12T14:17:35,268 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T14:17:35,269 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T14:17:35,269 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T14:17:35,271 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a0d122b7a4f4891082438eedd4480d0, UNASSIGN}] 2024-11-12T14:17:35,272 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a0d122b7a4f4891082438eedd4480d0, UNASSIGN 2024-11-12T14:17:35,272 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=46 updating hbase:meta row=1a0d122b7a4f4891082438eedd4480d0, regionState=CLOSING, regionLocation=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:35,274 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=46, ppid=45, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a0d122b7a4f4891082438eedd4480d0, UNASSIGN because future has completed 2024-11-12T14:17:35,274 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T14:17:35,274 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822}] 2024-11-12T14:17:35,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T14:17:35,426 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] handler.UnassignRegionHandler(122): Close 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:35,426 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T14:17:35,426 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1722): Closing 1a0d122b7a4f4891082438eedd4480d0, disabling compactions & flushes 2024-11-12T14:17:35,426 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:35,426 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:35,427 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. after waiting 0 ms 2024-11-12T14:17:35,427 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:35,427 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(2902): Flushing 1a0d122b7a4f4891082438eedd4480d0 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-12T14:17:35,427 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=A 2024-11-12T14:17:35,427 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:35,427 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=B 2024-11-12T14:17:35,427 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:35,427 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 1a0d122b7a4f4891082438eedd4480d0, store=C 2024-11-12T14:17:35,427 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:35,434 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ec917ce91fb94901ac596cfcd3bf3200_1a0d122b7a4f4891082438eedd4480d0 is 50, key is test_row_0/A:col10/1731421055253/Put/seqid=0 2024-11-12T14:17:35,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742039_1215 (size=9914) 2024-11-12T14:17:35,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T14:17:35,838 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:35,844 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ec917ce91fb94901ac596cfcd3bf3200_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ec917ce91fb94901ac596cfcd3bf3200_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:35,845 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/80ce927131a44bff93a57ae7cb84d8d7, store: [table=TestAcidGuarantees family=A region=1a0d122b7a4f4891082438eedd4480d0] 2024-11-12T14:17:35,846 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/80ce927131a44bff93a57ae7cb84d8d7 is 175, key is test_row_0/A:col10/1731421055253/Put/seqid=0 2024-11-12T14:17:35,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742040_1216 (size=22561) 2024-11-12T14:17:35,857 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=364, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/80ce927131a44bff93a57ae7cb84d8d7 2024-11-12T14:17:35,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T14:17:35,890 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/850ca61eed364a92982a0c29a5b8c407 is 50, key is test_row_0/B:col10/1731421055253/Put/seqid=0 2024-11-12T14:17:35,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742041_1217 (size=9857) 2024-11-12T14:17:36,298 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/850ca61eed364a92982a0c29a5b8c407 2024-11-12T14:17:36,306 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/cca6f2da44184b679d121a18b065edda is 50, key is test_row_0/C:col10/1731421055253/Put/seqid=0 2024-11-12T14:17:36,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742042_1218 (size=9857) 2024-11-12T14:17:36,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T14:17:36,723 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/cca6f2da44184b679d121a18b065edda 2024-11-12T14:17:36,729 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/A/80ce927131a44bff93a57ae7cb84d8d7 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/80ce927131a44bff93a57ae7cb84d8d7 2024-11-12T14:17:36,737 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/80ce927131a44bff93a57ae7cb84d8d7, entries=100, sequenceid=364, filesize=22.0 K 2024-11-12T14:17:36,738 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/B/850ca61eed364a92982a0c29a5b8c407 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/850ca61eed364a92982a0c29a5b8c407 2024-11-12T14:17:36,751 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/850ca61eed364a92982a0c29a5b8c407, entries=100, sequenceid=364, filesize=9.6 K 2024-11-12T14:17:36,755 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/.tmp/C/cca6f2da44184b679d121a18b065edda as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/cca6f2da44184b679d121a18b065edda 2024-11-12T14:17:36,770 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/cca6f2da44184b679d121a18b065edda, entries=100, sequenceid=364, filesize=9.6 K 2024-11-12T14:17:36,771 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 1a0d122b7a4f4891082438eedd4480d0 in 1344ms, sequenceid=364, compaction requested=true 2024-11-12T14:17:36,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/92c54b383125465ab5dacc3b17b6f287, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/5a588b28cc514ca8ada4254b4d3297b6, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/6dce43b416b741c8b3a745e3a3b9e17e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/76c150bd24144407a0da98be54542466, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/82be5972d5dd44c78bad539458e5d182, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/62c20cdbdb71436fb0135f5cb6c908b3, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/709143b76b084c3d9532d98b198b13e1, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/c225d06f910f4c8083c2fb087d65e7d5, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/b06db45f460e41d38102f458c67550a6, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/65fc04128fab481b9c772eaf2a2574b1, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/bedb384c020e4abd8be25ae325391c03, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/86fc31a312ac498f89ad1cf6801b767b, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/4640f9934ed049f2bcc53adc59b70c54, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/58c7fd3536754bb280af43c86d6c9cf6, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/d8e516d795b840f7b93751c7469041da, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/0e88359daf1a4e199e757cf8e46b99a0, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/780141815efd40c596b072b06afd2016, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/3bb35026f6a644889c668521c65968d7, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/472f8a1851b644d58a72b01681ea2328, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/f0c5e3ca377b4e13bdb081dc0e887463, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/5c5fa170a4184ea486c5855d81814a15, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/083b513989794a59bdb3c16bad8dd0c6, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/4db7f85eb73e4bd9b94de81bd2b6bc43, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/f02301ee8ff4494eb320faf9cfa3439a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/904486d3b86f400794b77c8f6f6170eb, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/3c2e6c0463e04f08b4783df2570e3eba] to archive 2024-11-12T14:17:36,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T14:17:36,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/92c54b383125465ab5dacc3b17b6f287 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/92c54b383125465ab5dacc3b17b6f287 2024-11-12T14:17:36,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/5a588b28cc514ca8ada4254b4d3297b6 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/5a588b28cc514ca8ada4254b4d3297b6 2024-11-12T14:17:36,789 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/6dce43b416b741c8b3a745e3a3b9e17e to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/6dce43b416b741c8b3a745e3a3b9e17e 2024-11-12T14:17:36,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/76c150bd24144407a0da98be54542466 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/76c150bd24144407a0da98be54542466 2024-11-12T14:17:36,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/82be5972d5dd44c78bad539458e5d182 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/82be5972d5dd44c78bad539458e5d182 2024-11-12T14:17:36,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/62c20cdbdb71436fb0135f5cb6c908b3 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/62c20cdbdb71436fb0135f5cb6c908b3 2024-11-12T14:17:36,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/709143b76b084c3d9532d98b198b13e1 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/709143b76b084c3d9532d98b198b13e1 2024-11-12T14:17:36,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/c225d06f910f4c8083c2fb087d65e7d5 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/c225d06f910f4c8083c2fb087d65e7d5 2024-11-12T14:17:36,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/b06db45f460e41d38102f458c67550a6 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/b06db45f460e41d38102f458c67550a6 2024-11-12T14:17:36,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/65fc04128fab481b9c772eaf2a2574b1 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/65fc04128fab481b9c772eaf2a2574b1 2024-11-12T14:17:36,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/bedb384c020e4abd8be25ae325391c03 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/bedb384c020e4abd8be25ae325391c03 2024-11-12T14:17:36,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/86fc31a312ac498f89ad1cf6801b767b to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/86fc31a312ac498f89ad1cf6801b767b 2024-11-12T14:17:36,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/4640f9934ed049f2bcc53adc59b70c54 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/4640f9934ed049f2bcc53adc59b70c54 2024-11-12T14:17:36,830 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/58c7fd3536754bb280af43c86d6c9cf6 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/58c7fd3536754bb280af43c86d6c9cf6 2024-11-12T14:17:36,831 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/d8e516d795b840f7b93751c7469041da to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/d8e516d795b840f7b93751c7469041da 2024-11-12T14:17:36,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/0e88359daf1a4e199e757cf8e46b99a0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/0e88359daf1a4e199e757cf8e46b99a0 2024-11-12T14:17:36,834 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/780141815efd40c596b072b06afd2016 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/780141815efd40c596b072b06afd2016 2024-11-12T14:17:36,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/3bb35026f6a644889c668521c65968d7 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/3bb35026f6a644889c668521c65968d7 2024-11-12T14:17:36,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/472f8a1851b644d58a72b01681ea2328 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/472f8a1851b644d58a72b01681ea2328 2024-11-12T14:17:36,837 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/f0c5e3ca377b4e13bdb081dc0e887463 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/f0c5e3ca377b4e13bdb081dc0e887463 2024-11-12T14:17:36,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/5c5fa170a4184ea486c5855d81814a15 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/5c5fa170a4184ea486c5855d81814a15 2024-11-12T14:17:36,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/083b513989794a59bdb3c16bad8dd0c6 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/083b513989794a59bdb3c16bad8dd0c6 2024-11-12T14:17:36,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/4db7f85eb73e4bd9b94de81bd2b6bc43 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/4db7f85eb73e4bd9b94de81bd2b6bc43 2024-11-12T14:17:36,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/f02301ee8ff4494eb320faf9cfa3439a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/f02301ee8ff4494eb320faf9cfa3439a 2024-11-12T14:17:36,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/904486d3b86f400794b77c8f6f6170eb to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/904486d3b86f400794b77c8f6f6170eb 2024-11-12T14:17:36,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/3c2e6c0463e04f08b4783df2570e3eba to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/3c2e6c0463e04f08b4783df2570e3eba 2024-11-12T14:17:36,852 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/53ed751256cf4adaa96f770f02d17b7a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/07dbfe027ad34b23b02b66314555a88c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/dbda84ef4a594ea5a9a0d51c275e71de, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/bf6f431ee86541c9b0cebf50e7045067, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/3710459c669b4bed84ba8bf9492e0ae5, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/0aa4a9a3203b47d9b9129cbbb08d015b, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/b0e4160f02eb42f083b1735ff28c1649, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/1c7723e1896047e9bda0e2d50cbf71ca, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/1b958dc5e57b4fce9078b45691122344, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/9674a4c0c6104eeba7d90ec90994cfd8, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/c0eb353038af4f9688d05723a2a2d388, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/a7658be41bba40ae8b58fdddd4f8d30a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/4d7c656486da4a509f53d8c444c55dfc, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/4692a48480b44eefbaa6feb6a62b0fac, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/e0b17b48cbdc4e91b6e80fd153a652ac, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/cdb67c34b56f4399b945f515b9095e89, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/235d746c25284f6a8274abff1a40f136, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/44c81b69daf6486fb9a8e2fc261e60fb, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/b547eb2c4e924243b50681f8a30f1e73, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/5560c90a10b443ae8ee2c4f2947fc91b, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/78be43a73cfd48c1bf3c36bb547e04d1, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/9696bad4295e408aa4e9af9f6a2a4030, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/bf3a33dd82a4462eb7e9a7a975e0794a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/bde243c885f54e4e8b4c0e11c29ea7c1, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/e61aa820d3fe4d5d837840262c0305ef, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/1c12884922c4422faeefb391270f13e6] to archive 2024-11-12T14:17:36,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T14:17:36,855 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/53ed751256cf4adaa96f770f02d17b7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/53ed751256cf4adaa96f770f02d17b7a 2024-11-12T14:17:36,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/07dbfe027ad34b23b02b66314555a88c to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/07dbfe027ad34b23b02b66314555a88c 2024-11-12T14:17:36,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/dbda84ef4a594ea5a9a0d51c275e71de to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/dbda84ef4a594ea5a9a0d51c275e71de 2024-11-12T14:17:36,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/bf6f431ee86541c9b0cebf50e7045067 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/bf6f431ee86541c9b0cebf50e7045067 2024-11-12T14:17:36,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/3710459c669b4bed84ba8bf9492e0ae5 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/3710459c669b4bed84ba8bf9492e0ae5 2024-11-12T14:17:36,881 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/0aa4a9a3203b47d9b9129cbbb08d015b to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/0aa4a9a3203b47d9b9129cbbb08d015b 2024-11-12T14:17:36,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/b0e4160f02eb42f083b1735ff28c1649 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/b0e4160f02eb42f083b1735ff28c1649 2024-11-12T14:17:36,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/1c7723e1896047e9bda0e2d50cbf71ca to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/1c7723e1896047e9bda0e2d50cbf71ca 2024-11-12T14:17:36,897 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/1b958dc5e57b4fce9078b45691122344 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/1b958dc5e57b4fce9078b45691122344 2024-11-12T14:17:36,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/9674a4c0c6104eeba7d90ec90994cfd8 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/9674a4c0c6104eeba7d90ec90994cfd8 2024-11-12T14:17:36,917 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/c0eb353038af4f9688d05723a2a2d388 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/c0eb353038af4f9688d05723a2a2d388 2024-11-12T14:17:36,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/a7658be41bba40ae8b58fdddd4f8d30a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/a7658be41bba40ae8b58fdddd4f8d30a 2024-11-12T14:17:36,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/4d7c656486da4a509f53d8c444c55dfc to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/4d7c656486da4a509f53d8c444c55dfc 2024-11-12T14:17:36,936 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/4692a48480b44eefbaa6feb6a62b0fac to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/4692a48480b44eefbaa6feb6a62b0fac 2024-11-12T14:17:36,938 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/e0b17b48cbdc4e91b6e80fd153a652ac to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/e0b17b48cbdc4e91b6e80fd153a652ac 2024-11-12T14:17:36,940 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/cdb67c34b56f4399b945f515b9095e89 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/cdb67c34b56f4399b945f515b9095e89 2024-11-12T14:17:36,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/235d746c25284f6a8274abff1a40f136 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/235d746c25284f6a8274abff1a40f136 2024-11-12T14:17:36,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/44c81b69daf6486fb9a8e2fc261e60fb to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/44c81b69daf6486fb9a8e2fc261e60fb 2024-11-12T14:17:36,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/b547eb2c4e924243b50681f8a30f1e73 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/b547eb2c4e924243b50681f8a30f1e73 2024-11-12T14:17:36,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/5560c90a10b443ae8ee2c4f2947fc91b to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/5560c90a10b443ae8ee2c4f2947fc91b 2024-11-12T14:17:36,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/78be43a73cfd48c1bf3c36bb547e04d1 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/78be43a73cfd48c1bf3c36bb547e04d1 2024-11-12T14:17:36,960 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/9696bad4295e408aa4e9af9f6a2a4030 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/9696bad4295e408aa4e9af9f6a2a4030 2024-11-12T14:17:36,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/bf3a33dd82a4462eb7e9a7a975e0794a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/bf3a33dd82a4462eb7e9a7a975e0794a 2024-11-12T14:17:36,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/bde243c885f54e4e8b4c0e11c29ea7c1 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/bde243c885f54e4e8b4c0e11c29ea7c1 2024-11-12T14:17:36,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/e61aa820d3fe4d5d837840262c0305ef to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/e61aa820d3fe4d5d837840262c0305ef 2024-11-12T14:17:36,970 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/1c12884922c4422faeefb391270f13e6 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/1c12884922c4422faeefb391270f13e6 2024-11-12T14:17:36,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/5e34deec33b543c9bda6ad7c6384c0d9, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/774ec1c8d09f4bed9faffca6305f61c4, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/73681badb6e14a31987ebe4e137efdad, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/fc58525cc6d545e6b24862e1771c3a8d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/dc28e453cb194ea79b125fecb4d52db6, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/ee31394860704199890769e87985e267, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/99af7a02f2c94119800665b944284cb1, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/2573157dd946408a9bbab98a2aabc5dc, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/b449676e3a9a434bb9b2b0c9985289b9, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/43a28dcd73834bc58cdb7d33afbc9931, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/baa31e18942447efb8d1110804e67930, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/89d7df7508d84892b5ba64a8d8a6331b, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/1eb19deb2b85439b952349eb92cc1260, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/59d2a7624ad645eb9b7dbf335ec9e91c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/d479e7fdd9d7408797a97d078455be65, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/0b7765b511864df2b41b82403e8179e3, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/fb0a7f0be2be40c1bce70719a8e1192d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/8528447e49b14adebdbc395d0859de94, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/54c70a1730d147bdbc8306e939830c59, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/6cb6ca90d3f944f58f90c6737cab93a3, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/f9ab4b2a5d954a9eb6f240954434155d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/89f6ff559a014c4bb10cd9546825221e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/bb6a2e5cf4e44b949872e0eda34e8a45, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/0ac0726e686e40c495f65e973e1f675b, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/a5205d7be1d9404590c035499ee1165d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/60d06e821f8f489b8d2c3444d5ffd6ce] to archive 2024-11-12T14:17:36,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T14:17:36,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/5e34deec33b543c9bda6ad7c6384c0d9 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/5e34deec33b543c9bda6ad7c6384c0d9 2024-11-12T14:17:36,986 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/774ec1c8d09f4bed9faffca6305f61c4 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/774ec1c8d09f4bed9faffca6305f61c4 2024-11-12T14:17:36,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/73681badb6e14a31987ebe4e137efdad to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/73681badb6e14a31987ebe4e137efdad 2024-11-12T14:17:36,996 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/fc58525cc6d545e6b24862e1771c3a8d to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/fc58525cc6d545e6b24862e1771c3a8d 2024-11-12T14:17:37,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/dc28e453cb194ea79b125fecb4d52db6 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/dc28e453cb194ea79b125fecb4d52db6 2024-11-12T14:17:37,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/ee31394860704199890769e87985e267 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/ee31394860704199890769e87985e267 2024-11-12T14:17:37,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/99af7a02f2c94119800665b944284cb1 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/99af7a02f2c94119800665b944284cb1 2024-11-12T14:17:37,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/2573157dd946408a9bbab98a2aabc5dc to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/2573157dd946408a9bbab98a2aabc5dc 2024-11-12T14:17:37,019 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/b449676e3a9a434bb9b2b0c9985289b9 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/b449676e3a9a434bb9b2b0c9985289b9 2024-11-12T14:17:37,022 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/43a28dcd73834bc58cdb7d33afbc9931 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/43a28dcd73834bc58cdb7d33afbc9931 2024-11-12T14:17:37,023 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/baa31e18942447efb8d1110804e67930 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/baa31e18942447efb8d1110804e67930 2024-11-12T14:17:37,025 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/89d7df7508d84892b5ba64a8d8a6331b to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/89d7df7508d84892b5ba64a8d8a6331b 2024-11-12T14:17:37,031 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/1eb19deb2b85439b952349eb92cc1260 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/1eb19deb2b85439b952349eb92cc1260 2024-11-12T14:17:37,033 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/59d2a7624ad645eb9b7dbf335ec9e91c to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/59d2a7624ad645eb9b7dbf335ec9e91c 2024-11-12T14:17:37,035 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/d479e7fdd9d7408797a97d078455be65 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/d479e7fdd9d7408797a97d078455be65 2024-11-12T14:17:37,037 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/0b7765b511864df2b41b82403e8179e3 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/0b7765b511864df2b41b82403e8179e3 2024-11-12T14:17:37,038 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/fb0a7f0be2be40c1bce70719a8e1192d to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/fb0a7f0be2be40c1bce70719a8e1192d 2024-11-12T14:17:37,039 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/8528447e49b14adebdbc395d0859de94 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/8528447e49b14adebdbc395d0859de94 2024-11-12T14:17:37,041 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/54c70a1730d147bdbc8306e939830c59 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/54c70a1730d147bdbc8306e939830c59 2024-11-12T14:17:37,043 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/6cb6ca90d3f944f58f90c6737cab93a3 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/6cb6ca90d3f944f58f90c6737cab93a3 2024-11-12T14:17:37,044 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/f9ab4b2a5d954a9eb6f240954434155d to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/f9ab4b2a5d954a9eb6f240954434155d 2024-11-12T14:17:37,047 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/89f6ff559a014c4bb10cd9546825221e to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/89f6ff559a014c4bb10cd9546825221e 2024-11-12T14:17:37,049 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/bb6a2e5cf4e44b949872e0eda34e8a45 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/bb6a2e5cf4e44b949872e0eda34e8a45 2024-11-12T14:17:37,053 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/0ac0726e686e40c495f65e973e1f675b to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/0ac0726e686e40c495f65e973e1f675b 2024-11-12T14:17:37,057 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/a5205d7be1d9404590c035499ee1165d to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/a5205d7be1d9404590c035499ee1165d 2024-11-12T14:17:37,064 DEBUG [StoreCloser-TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/60d06e821f8f489b8d2c3444d5ffd6ce to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/60d06e821f8f489b8d2c3444d5ffd6ce 2024-11-12T14:17:37,090 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/recovered.edits/367.seqid, newMaxSeqId=367, maxSeqId=4 2024-11-12T14:17:37,092 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. 2024-11-12T14:17:37,092 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] regionserver.HRegion(1676): Region close journal for 1a0d122b7a4f4891082438eedd4480d0: Waiting for close lock at 1731421055426Running coprocessor pre-close hooks at 1731421055426Disabling compacts and flushes for region at 1731421055426Disabling writes for close at 1731421055427 (+1 ms)Obtaining lock to block concurrent updates at 1731421055427Preparing flush snapshotting stores in 1a0d122b7a4f4891082438eedd4480d0 at 1731421055427Finished memstore snapshotting TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731421055427Flushing stores of TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0. at 1731421055427Flushing 1a0d122b7a4f4891082438eedd4480d0/A: creating writer at 1731421055428 (+1 ms)Flushing mob file 1a0d122b7a4f4891082438eedd4480d0/A: appending metadata at 1731421055433 (+5 ms)Flushing mob file 1a0d122b7a4f4891082438eedd4480d0/A: closing flushed file at 1731421055433Flushing 1a0d122b7a4f4891082438eedd4480d0/A: appending metadata at 1731421055845 (+412 ms)Flushing 1a0d122b7a4f4891082438eedd4480d0/A: closing flushed file at 1731421055845Flushing 1a0d122b7a4f4891082438eedd4480d0/B: creating writer at 1731421055865 (+20 ms)Flushing 1a0d122b7a4f4891082438eedd4480d0/B: appending metadata at 1731421055889 (+24 ms)Flushing 1a0d122b7a4f4891082438eedd4480d0/B: closing flushed file at 1731421055889Flushing 1a0d122b7a4f4891082438eedd4480d0/C: creating writer at 1731421056303 (+414 ms)Flushing 1a0d122b7a4f4891082438eedd4480d0/C: appending metadata at 1731421056306 (+3 ms)Flushing 1a0d122b7a4f4891082438eedd4480d0/C: closing flushed file at 1731421056306Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@f00d86e: reopening flushed file at 1731421056728 (+422 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79d0eb39: reopening flushed file at 1731421056737 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c5682f0: reopening flushed file at 1731421056751 (+14 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 1a0d122b7a4f4891082438eedd4480d0 in 1344ms, sequenceid=364, compaction requested=true at 1731421056772 (+21 ms)Writing region close event to WAL at 1731421057074 (+302 ms)Running coprocessor post-close hooks at 1731421057091 (+17 ms)Closed at 1731421057092 (+1 ms) 2024-11-12T14:17:37,106 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=46 updating hbase:meta row=1a0d122b7a4f4891082438eedd4480d0, regionState=CLOSED 2024-11-12T14:17:37,106 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=47}] handler.UnassignRegionHandler(157): Closed 1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:37,111 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=47, ppid=46, state=RUNNABLE, hasLock=false; CloseRegionProcedure 1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 because future has completed 2024-11-12T14:17:37,115 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-12T14:17:37,115 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; CloseRegionProcedure 1a0d122b7a4f4891082438eedd4480d0, server=2b6d221c5cde,42925,1731420990822 in 1.8390 sec 2024-11-12T14:17:37,121 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=46, resume processing ppid=45 2024-11-12T14:17:37,121 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, ppid=45, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=1a0d122b7a4f4891082438eedd4480d0, UNASSIGN in 1.8440 sec 2024-11-12T14:17:37,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-12T14:17:37,128 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8550 sec 2024-11-12T14:17:37,131 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731421057130"}]},"ts":"1731421057130"} 2024-11-12T14:17:37,133 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T14:17:37,133 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T14:17:37,138 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8720 sec 2024-11-12T14:17:37,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-12T14:17:37,410 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:17:37,410 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-12T14:17:37,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:17:37,413 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=48, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:17:37,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T14:17:37,415 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=48, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:17:37,418 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:37,440 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A, FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B, FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C, FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/recovered.edits] 2024-11-12T14:17:37,449 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/80ce927131a44bff93a57ae7cb84d8d7 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/80ce927131a44bff93a57ae7cb84d8d7 2024-11-12T14:17:37,454 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/abdff07dc4904b8f834b4525f829fc27 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/abdff07dc4904b8f834b4525f829fc27 2024-11-12T14:17:37,458 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/afeeefee6fef40a9b9437612e99e85a3 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/A/afeeefee6fef40a9b9437612e99e85a3 2024-11-12T14:17:37,466 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/18b41272526641f488a48c048cdfcb23 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/18b41272526641f488a48c048cdfcb23 2024-11-12T14:17:37,470 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/2b70b9b6e4564ed585e3b23506cb8639 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/2b70b9b6e4564ed585e3b23506cb8639 2024-11-12T14:17:37,474 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/850ca61eed364a92982a0c29a5b8c407 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/B/850ca61eed364a92982a0c29a5b8c407 2024-11-12T14:17:37,497 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/2e7397c2346e4c97851b1501fda220d7 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/2e7397c2346e4c97851b1501fda220d7 2024-11-12T14:17:37,499 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/807d2354f5ff4caca889f1d809ab0d80 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/807d2354f5ff4caca889f1d809ab0d80 2024-11-12T14:17:37,502 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/cca6f2da44184b679d121a18b065edda to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/C/cca6f2da44184b679d121a18b065edda 2024-11-12T14:17:37,506 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/recovered.edits/367.seqid to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0/recovered.edits/367.seqid 2024-11-12T14:17:37,513 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:37,513 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T14:17:37,514 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T14:17:37,518 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-12T14:17:37,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T14:17:37,530 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111203b96a7f066e4581a67e0665c9d7aefc_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111203b96a7f066e4581a67e0665c9d7aefc_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:37,533 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111214919e0b1cec4087b4ea8649c0c294fe_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111214919e0b1cec4087b4ea8649c0c294fe_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:37,538 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122d412a8196e74ffab7a4fd88899c2d98_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122d412a8196e74ffab7a4fd88899c2d98_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:37,542 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124168c26f70cb4398bb7268779b2f8c89_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124168c26f70cb4398bb7268779b2f8c89_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:37,545 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124a754f01b94b47cd9d8531023a96ea5d_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124a754f01b94b47cd9d8531023a96ea5d_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:37,548 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111259a9e9b596974d6dbc038837485bb602_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111259a9e9b596974d6dbc038837485bb602_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:37,550 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125f9c45c0fa5b42359baa9bd343de60cc_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125f9c45c0fa5b42359baa9bd343de60cc_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:37,555 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411128075adfb52e04115b99404267f907ce5_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411128075adfb52e04115b99404267f907ce5_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:37,570 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112811028c80cb9475a952b647275d7ffc4_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112811028c80cb9475a952b647275d7ffc4_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:37,581 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111285669a7aee6e48299460570616b4d9dd_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111285669a7aee6e48299460570616b4d9dd_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:37,597 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411128598e75f551248179685c142f97f7f25_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411128598e75f551248179685c142f97f7f25_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:37,606 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112886a677a5e7944578267572ac31e045d_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112886a677a5e7944578267572ac31e045d_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:37,620 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129efa0d70edff4501abdfc24135e57dab_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129efa0d70edff4501abdfc24135e57dab_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:37,629 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a9f802b91bb841708cfc063e32506037_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112a9f802b91bb841708cfc063e32506037_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:37,639 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b830e3a204764affae111236b1ec6655_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b830e3a204764affae111236b1ec6655_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:37,649 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c20e8ac2fea44b8d8f0186c4bbb6568c_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c20e8ac2fea44b8d8f0186c4bbb6568c_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:37,655 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d9a6a3f04a3645e89203101a4da61761_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d9a6a3f04a3645e89203101a4da61761_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:37,674 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112df5cb7b9bddc450aaf7409b869faeed8_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112df5cb7b9bddc450aaf7409b869faeed8_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:37,688 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ebe063ebc57e4d9fac09de7719d4864d_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ebe063ebc57e4d9fac09de7719d4864d_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:37,691 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ec917ce91fb94901ac596cfcd3bf3200_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ec917ce91fb94901ac596cfcd3bf3200_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:37,698 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f5a571fb856d4df495978057c0593937_1a0d122b7a4f4891082438eedd4480d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f5a571fb856d4df495978057c0593937_1a0d122b7a4f4891082438eedd4480d0 2024-11-12T14:17:37,701 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T14:17:37,704 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=48, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:17:37,707 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T14:17:37,709 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T14:17:37,711 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=48, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:17:37,711 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T14:17:37,711 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731421057711"}]},"ts":"9223372036854775807"} 2024-11-12T14:17:37,714 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T14:17:37,714 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 1a0d122b7a4f4891082438eedd4480d0, NAME => 'TestAcidGuarantees,,1731421023476.1a0d122b7a4f4891082438eedd4480d0.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T14:17:37,714 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T14:17:37,714 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731421057714"}]},"ts":"9223372036854775807"} 2024-11-12T14:17:37,717 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T14:17:37,718 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=48, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:17:37,719 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 308 msec 2024-11-12T14:17:37,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-12T14:17:37,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T14:17:37,730 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:17:37,743 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobMixedAtomicity Thread=248 (was 249), OpenFileDescriptor=451 (was 457), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1242 (was 1082) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=499 (was 1068) 2024-11-12T14:17:37,754 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=248, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=1242, ProcessCount=11, AvailableMemoryMB=497 2024-11-12T14:17:37,758 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T14:17:37,758 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T14:17:37,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=49, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T14:17:37,773 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=49, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T14:17:37,773 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:37,773 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 49 2024-11-12T14:17:37,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49 2024-11-12T14:17:37,775 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=49, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T14:17:37,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742043_1219 (size=1061) 2024-11-12T14:17:37,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49 2024-11-12T14:17:38,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49 2024-11-12T14:17:38,194 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3 2024-11-12T14:17:38,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742044_1220 (size=53) 2024-11-12T14:17:38,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49 2024-11-12T14:17:38,626 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:17:38,627 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 7a67a89740591358c1149d6b779d0225, disabling compactions & flushes 2024-11-12T14:17:38,627 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:38,627 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:38,627 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. after waiting 0 ms 2024-11-12T14:17:38,627 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:38,627 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:38,627 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 7a67a89740591358c1149d6b779d0225: Waiting for close lock at 1731421058626Disabling compacts and flushes for region at 1731421058626Disabling writes for close at 1731421058627 (+1 ms)Writing region close event to WAL at 1731421058627Closed at 1731421058627 2024-11-12T14:17:38,628 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=49, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T14:17:38,628 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731421058628"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731421058628"}]},"ts":"1731421058628"} 2024-11-12T14:17:38,630 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T14:17:38,631 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=49, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T14:17:38,631 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731421058631"}]},"ts":"1731421058631"} 2024-11-12T14:17:38,633 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T14:17:38,633 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7a67a89740591358c1149d6b779d0225, ASSIGN}] 2024-11-12T14:17:38,635 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7a67a89740591358c1149d6b779d0225, ASSIGN 2024-11-12T14:17:38,635 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7a67a89740591358c1149d6b779d0225, ASSIGN; state=OFFLINE, location=2b6d221c5cde,42925,1731420990822; forceNewPlan=false, retain=false 2024-11-12T14:17:38,786 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=7a67a89740591358c1149d6b779d0225, regionState=OPENING, regionLocation=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:38,788 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=50, ppid=49, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7a67a89740591358c1149d6b779d0225, ASSIGN because future has completed 2024-11-12T14:17:38,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822}] 2024-11-12T14:17:38,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49 2024-11-12T14:17:38,944 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:38,944 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7752): Opening region: {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:17:38,945 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:38,945 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:17:38,945 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7794): checking encryption for 7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:38,945 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(7797): checking classloading for 7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:38,946 INFO [StoreOpener-7a67a89740591358c1149d6b779d0225-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:38,948 INFO [StoreOpener-7a67a89740591358c1149d6b779d0225-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T14:17:38,948 INFO [StoreOpener-7a67a89740591358c1149d6b779d0225-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a67a89740591358c1149d6b779d0225 columnFamilyName A 2024-11-12T14:17:38,948 DEBUG [StoreOpener-7a67a89740591358c1149d6b779d0225-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:38,949 INFO [StoreOpener-7a67a89740591358c1149d6b779d0225-1 {}] regionserver.HStore(327): Store=7a67a89740591358c1149d6b779d0225/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:17:38,949 INFO [StoreOpener-7a67a89740591358c1149d6b779d0225-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:38,951 INFO [StoreOpener-7a67a89740591358c1149d6b779d0225-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T14:17:38,952 INFO [StoreOpener-7a67a89740591358c1149d6b779d0225-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a67a89740591358c1149d6b779d0225 columnFamilyName B 2024-11-12T14:17:38,952 DEBUG [StoreOpener-7a67a89740591358c1149d6b779d0225-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:38,952 INFO [StoreOpener-7a67a89740591358c1149d6b779d0225-1 {}] regionserver.HStore(327): Store=7a67a89740591358c1149d6b779d0225/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:17:38,952 INFO [StoreOpener-7a67a89740591358c1149d6b779d0225-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:38,953 INFO [StoreOpener-7a67a89740591358c1149d6b779d0225-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T14:17:38,954 INFO [StoreOpener-7a67a89740591358c1149d6b779d0225-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 7a67a89740591358c1149d6b779d0225 columnFamilyName C 2024-11-12T14:17:38,954 DEBUG [StoreOpener-7a67a89740591358c1149d6b779d0225-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:17:38,954 INFO [StoreOpener-7a67a89740591358c1149d6b779d0225-1 {}] regionserver.HStore(327): Store=7a67a89740591358c1149d6b779d0225/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:17:38,954 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:38,954 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1038): replaying wal for 7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:38,956 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:38,957 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:38,958 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1048): stopping wal replay for 7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:38,958 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1060): Cleaning up temporary data for 7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:38,959 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T14:17:38,960 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1093): writing seq id for 7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:38,966 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:17:38,966 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1114): Opened 7a67a89740591358c1149d6b779d0225; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64136039, jitterRate=-0.04429854452610016}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T14:17:38,966 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:38,967 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegion(1006): Region open journal for 7a67a89740591358c1149d6b779d0225: Running coprocessor pre-open hook at 1731421058945Writing region info on filesystem at 1731421058945Initializing all the Stores at 1731421058946 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731421058946Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731421058946Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731421058946Cleaning up temporary data from old regions at 1731421058958 (+12 ms)Running coprocessor post-open hooks at 1731421058966 (+8 ms)Region opened successfully at 1731421058967 (+1 ms) 2024-11-12T14:17:38,968 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., pid=51, masterSystemTime=1731421058941 2024-11-12T14:17:38,974 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:38,974 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=51}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:38,975 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=50 updating hbase:meta row=7a67a89740591358c1149d6b779d0225, regionState=OPEN, openSeqNum=2, regionLocation=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:38,980 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=51, ppid=50, state=RUNNABLE, hasLock=false; OpenRegionProcedure 7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 because future has completed 2024-11-12T14:17:38,986 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-12T14:17:38,986 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; OpenRegionProcedure 7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 in 194 msec 2024-11-12T14:17:38,988 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=50, resume processing ppid=49 2024-11-12T14:17:38,988 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, ppid=49, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7a67a89740591358c1149d6b779d0225, ASSIGN in 353 msec 2024-11-12T14:17:38,989 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=49, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T14:17:38,989 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731421058989"}]},"ts":"1731421058989"} 2024-11-12T14:17:38,992 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T14:17:38,994 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=49, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T14:17:38,996 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2360 sec 2024-11-12T14:17:39,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=49 2024-11-12T14:17:39,920 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:17:39,921 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@533c1a83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:39,921 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:17:39,922 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:17:39,923 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:17:39,923 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:17:39,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:17:39,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@325ce91e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:39,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:17:39,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:17:39,924 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:39,926 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36674, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:17:39,927 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@208224cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:39,927 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:17:39,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:17:39,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:17:39,931 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49122, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:17:39,932 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T14:17:39,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 2b6d221c5cde,45591,1731420989939 2024-11-12T14:17:39,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@10dc1304 2024-11-12T14:17:39,933 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T14:17:39,934 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36684, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T14:17:39,936 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b2dccd0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:39,936 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:17:39,936 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:17:39,940 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:17:39,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:17:39,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:17:39,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c99bee1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:39,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:17:39,941 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:17:39,942 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:39,942 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36704, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:17:39,943 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6daf2f58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:39,960 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@661204ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:39,961 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:17:39,961 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:17:39,961 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:17:39,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:17:39,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:17:39,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5cada3be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:39,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:17:39,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:17:39,962 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:39,963 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:17:39,964 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70b59964, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:39,965 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40655b61, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:39,965 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:17:39,965 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:17:39,966 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:17:39,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:17:39,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:17:39,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@210c2ea3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:39,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:17:39,966 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:17:39,967 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:39,967 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36740, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:17:39,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@732f6fe7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:39,972 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d355d6c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:39,972 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:17:39,973 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:17:39,974 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:17:39,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:17:39,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:17:39,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1398087, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:39,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:17:39,974 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:17:39,975 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:39,976 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:17:39,977 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34947ffc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:39,977 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1cbf4401, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:39,977 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:17:39,978 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:17:39,979 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:17:39,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:17:39,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:17:39,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4871e268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:39,979 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:17:39,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:17:39,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:39,981 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36780, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:17:39,981 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cca509d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:39,984 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a987c8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:39,984 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:17:39,985 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:17:39,986 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:17:39,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:17:39,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:17:39,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62e306ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:39,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:17:39,986 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:17:39,987 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:39,988 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36788, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:17:39,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26bbc75b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:39,992 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a60f526, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:39,992 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:17:39,993 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:17:39,993 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:17:39,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:17:39,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:17:39,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43541604, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:39,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:17:39,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:17:39,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:39,995 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:17:39,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57cb86e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:40,008 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fd078e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:40,008 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:17:40,009 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:17:40,010 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:17:40,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:17:40,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:17:40,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@755924be, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:40,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:17:40,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:17:40,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:40,012 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36840, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:17:40,012 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4389e259, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:40,013 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@515ac970, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:40,013 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:17:40,013 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:17:40,014 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:17:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:17:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:17:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@944b81d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:17:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:17:40,015 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:40,016 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36854, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:17:40,017 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63114d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:40,022 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62a38ab9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:40,022 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:17:40,022 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:17:40,023 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:17:40,023 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:17:40,023 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:17:40,023 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a1ecd2b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:40,023 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:17:40,023 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:17:40,024 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:17:40,024 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:36874, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:17:40,025 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2828aa04, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:17:40,029 DEBUG [Thread-1057 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:17:40,029 DEBUG [Thread-1055 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:17:40,029 DEBUG [Thread-1042 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:17:40,029 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:17:40,029 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:17:40,029 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:17:40,030 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:17:40,030 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:17:40,030 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:17:40,031 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49142, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:17:40,031 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49132, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:17:40,031 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49148, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:17:40,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2] 2024-11-12T14:17:40,034 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2] 2024-11-12T14:17:40,035 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2] 2024-11-12T14:17:40,037 DEBUG [Thread-1059 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:17:40,038 DEBUG [Thread-1059 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:17:40,038 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:17:40,038 DEBUG [Thread-1059 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:17:40,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-12T14:17:40,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T14:17:40,040 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49154, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:17:40,041 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:17:40,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2] 2024-11-12T14:17:40,042 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:17:40,043 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:17:40,048 DEBUG [Thread-1044 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:17:40,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:17:40,049 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:17:40,051 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49170, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:17:40,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2] 2024-11-12T14:17:40,055 DEBUG [Thread-1051 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:17:40,055 DEBUG [Thread-1046 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:17:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:17:40,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:17:40,057 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49174, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:17:40,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:40,059 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7a67a89740591358c1149d6b779d0225 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T14:17:40,059 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2] 2024-11-12T14:17:40,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=A 2024-11-12T14:17:40,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:40,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=B 2024-11-12T14:17:40,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:40,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=C 2024-11-12T14:17:40,060 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:40,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:17:40,060 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:17:40,063 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49176, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:17:40,065 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2] 2024-11-12T14:17:40,068 DEBUG [Thread-1040 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:17:40,070 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:17:40,070 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:17:40,071 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:17:40,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2] 2024-11-12T14:17:40,077 DEBUG [Thread-1048 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:17:40,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:17:40,078 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:17:40,080 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49198, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:17:40,081 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2] 2024-11-12T14:17:40,088 DEBUG [Thread-1053 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:17:40,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:17:40,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:17:40,091 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:49210, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:17:40,093 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2] 2024-11-12T14:17:40,109 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/35ae6d1dee954d0d9ac630030115b152 is 50, key is test_row_0/A:col10/1731421060058/Put/seqid=0 2024-11-12T14:17:40,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731421060101, max timestamp=1731421060122Num uniques 859; 2024-11-12T14:17:40,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T14:17:40,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731421060101, max timestamp=1731421060122Num uniques 859; 2024-11-12T14:17:40,129 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T14:17:40,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731421060101, max timestamp=1731421060122Num uniques 859; 2024-11-12T14:17:40,132 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T14:17:40,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T14:17:40,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742045_1221 (size=12001) 2024-11-12T14:17:40,158 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/35ae6d1dee954d0d9ac630030115b152 2024-11-12T14:17:40,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=218, cellSize=9978, totalHeapSize=22912, min timestamp=1731421060162, max timestamp=1731421060176Num uniques 1018; 2024-11-12T14:17:40,181 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=9978, suffix heap size=133840,new segment heap size=22912 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=218 2024-11-12T14:17:40,186 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731421060162, max timestamp=1731421060176Num uniques 1018; 2024-11-12T14:17:40,186 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-12T14:17:40,190 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731421060162, max timestamp=1731421060176Num uniques 1018; 2024-11-12T14:17:40,190 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-12T14:17:40,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T14:17:40,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:40,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:40,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:40,208 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:40,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:40,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:40,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421120219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,238 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:40,241 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:40,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421120223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:40,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:40,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731421060193, max timestamp=1731421060232Num uniques 1077; 2024-11-12T14:17:40,244 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49322, new segment data size=10392, suffix heap size=139976,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1077, new segment cells count=227 2024-11-12T14:17:40,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731421060193, max timestamp=1731421060232Num uniques 1027; 2024-11-12T14:17:40,247 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=6870, suffix heap size=134776,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=150 2024-11-12T14:17:40,251 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731421060193, max timestamp=1731421060232Num uniques 1027; 2024-11-12T14:17:40,251 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=6870, suffix heap size=134776,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=150 2024-11-12T14:17:40,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/69946287a9eb4ddaada2e2959c0a7774 is 50, key is test_row_0/B:col10/1731421060058/Put/seqid=0 2024-11-12T14:17:40,267 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742046_1222 (size=12001) 2024-11-12T14:17:40,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/69946287a9eb4ddaada2e2959c0a7774 2024-11-12T14:17:40,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:40,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421120306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:40,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,324 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:40,324 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:40,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421120315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:40,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:40,325 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:40,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421120317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:40,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,326 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:40,354 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:40,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421120349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:40,355 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:40,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421120349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:40,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:40,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:40,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T14:17:40,373 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T14:17:40,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:40,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:40,373 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:40,373 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:40,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:40,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:40,380 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/32d725af0bbb4b25ae8b71e7021d07af is 50, key is test_row_0/C:col10/1731421060058/Put/seqid=0 2024-11-12T14:17:40,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742047_1223 (size=12001) 2024-11-12T14:17:40,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:40,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421120430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:40,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,438 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:40,438 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:40,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421120430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:40,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,439 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:40,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:40,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421120430, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:40,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,439 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:40,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T14:17:40,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:40,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:40,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:40,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:40,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:40,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:40,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:40,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421120559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:40,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:40,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:40,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421120570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:40,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,575 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:40,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:40,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421120650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:40,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:40,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421120650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:40,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421120650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,653 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:40,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:40,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T14:17:40,699 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T14:17:40,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:40,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:40,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:40,700 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:40,701 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:40,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:40,836 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=11 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/32d725af0bbb4b25ae8b71e7021d07af 2024-11-12T14:17:40,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T14:17:40,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:40,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:40,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:40,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:40,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:40,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:40,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:40,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421120869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:40,875 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,876 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:40,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:40,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421120880, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:40,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:40,884 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:40,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/35ae6d1dee954d0d9ac630030115b152 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/35ae6d1dee954d0d9ac630030115b152 2024-11-12T14:17:40,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/35ae6d1dee954d0d9ac630030115b152, entries=150, sequenceid=11, filesize=11.7 K 2024-11-12T14:17:40,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/69946287a9eb4ddaada2e2959c0a7774 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/69946287a9eb4ddaada2e2959c0a7774 2024-11-12T14:17:40,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/69946287a9eb4ddaada2e2959c0a7774, entries=150, sequenceid=11, filesize=11.7 K 2024-11-12T14:17:40,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/32d725af0bbb4b25ae8b71e7021d07af as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/32d725af0bbb4b25ae8b71e7021d07af 2024-11-12T14:17:40,958 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/32d725af0bbb4b25ae8b71e7021d07af, entries=150, sequenceid=11, filesize=11.7 K 2024-11-12T14:17:40,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.01 KB/137226 for 7a67a89740591358c1149d6b779d0225 in 910ms, sequenceid=11, compaction requested=false 2024-11-12T14:17:40,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:40,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731421060290, max timestamp=1731421060965Num uniques 1086; 2024-11-12T14:17:40,972 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-12T14:17:40,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731421060290, max timestamp=1731421060965Num uniques 1009; 2024-11-12T14:17:40,975 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8516, suffix heap size=132904,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=186 2024-11-12T14:17:40,976 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731421060290, max timestamp=1731421060965Num uniques 1009; 2024-11-12T14:17:40,976 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8516, suffix heap size=132904,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=186 2024-11-12T14:17:40,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:40,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7a67a89740591358c1149d6b779d0225 3/3 column families, dataSize=53.67 KB heapSize=143.39 KB 2024-11-12T14:17:40,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=A 2024-11-12T14:17:40,984 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:17:40,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=B 2024-11-12T14:17:40,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:17:40,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=C 2024-11-12T14:17:40,985 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:17:41,004 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/b00ff763a1cb499eb2548ffbe473a361 is 50, key is test_row_0/A:col10/1731421060980/Put/seqid=0 2024-11-12T14:17:41,013 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T14:17:41,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:41,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:41,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:41,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:41,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:41,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:41,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742048_1224 (size=14341) 2024-11-12T14:17:41,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/b00ff763a1cb499eb2548ffbe473a361 2024-11-12T14:17:41,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/c15c09973b9246ccbd052d5cf9314784 is 50, key is test_row_0/B:col10/1731421060980/Put/seqid=0 2024-11-12T14:17:41,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:41,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421121050, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:41,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:41,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731421061021, max timestamp=1731421061055Num uniques 859; 2024-11-12T14:17:41,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731421061021, max timestamp=1731421061055Num uniques 859; 2024-11-12T14:17:41,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:41,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:41,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T14:17:41,059 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T14:17:41,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731421061021, max timestamp=1731421061055Num uniques 859; 2024-11-12T14:17:41,062 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T14:17:41,072 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742049_1225 (size=12001) 2024-11-12T14:17:41,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:41,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421121164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:41,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:41,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:41,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:41,171 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:41,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421121164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:41,172 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:41,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T14:17:41,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:41,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:41,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:41,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:41,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:41,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:41,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421121169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:41,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:41,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:41,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:41,175 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:41,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T14:17:41,285 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:41,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421121282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:41,286 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:41,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:41,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:41,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421121282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:41,286 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:41,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:41,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:41,286 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:41,327 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T14:17:41,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:41,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:41,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:41,328 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:41,328 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:41,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:41,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421121384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:41,387 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:41,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421121385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:41,387 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:41,388 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:41,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:41,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421121390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:41,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:41,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:41,392 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:41,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T14:17:41,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:41,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:41,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:41,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:41,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:41,492 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/c15c09973b9246ccbd052d5cf9314784 2024-11-12T14:17:41,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:41,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:41,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421121491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:41,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:41,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:41,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:41,504 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:41,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421121499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:41,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:41,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:41,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:41,528 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/1cb3e579dc114cc0bf97244f38a391aa is 50, key is test_row_0/C:col10/1731421060980/Put/seqid=0 2024-11-12T14:17:41,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742050_1226 (size=12001) 2024-11-12T14:17:41,647 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T14:17:41,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:41,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:41,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:41,647 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:41,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:41,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:41,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:41,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421121699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:41,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:41,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:41,702 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:41,800 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T14:17:41,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:41,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:41,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:41,801 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:41,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:41,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:41,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:41,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421121811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:41,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:41,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:41,816 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:41,821 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:41,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421121819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:41,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:41,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:41,821 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:41,946 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/1cb3e579dc114cc0bf97244f38a391aa 2024-11-12T14:17:41,962 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/b00ff763a1cb499eb2548ffbe473a361 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/b00ff763a1cb499eb2548ffbe473a361 2024-11-12T14:17:41,975 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/b00ff763a1cb499eb2548ffbe473a361, entries=200, sequenceid=88, filesize=14.0 K 2024-11-12T14:17:41,977 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T14:17:41,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:41,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:41,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:41,977 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:41,977 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:41,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/c15c09973b9246ccbd052d5cf9314784 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/c15c09973b9246ccbd052d5cf9314784 2024-11-12T14:17:41,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:41,985 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/c15c09973b9246ccbd052d5cf9314784, entries=150, sequenceid=88, filesize=11.7 K 2024-11-12T14:17:41,989 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/1cb3e579dc114cc0bf97244f38a391aa as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/1cb3e579dc114cc0bf97244f38a391aa 2024-11-12T14:17:41,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/1cb3e579dc114cc0bf97244f38a391aa, entries=150, sequenceid=88, filesize=11.7 K 2024-11-12T14:17:42,000 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~163.27 KB/167184, currentSize=134.18 KB/137400 for 7a67a89740591358c1149d6b779d0225 in 1016ms, sequenceid=88, compaction requested=false 2024-11-12T14:17:42,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:42,135 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-12T14:17:42,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:42,136 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 7a67a89740591358c1149d6b779d0225 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-12T14:17:42,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=A 2024-11-12T14:17:42,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:17:42,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=B 2024-11-12T14:17:42,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:17:42,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=C 2024-11-12T14:17:42,136 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:17:42,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/0ebded243c2d45d0a580f037a8e0b9c5 is 50, key is test_row_0/A:col10/1731421061139/Put/seqid=0 2024-11-12T14:17:42,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T14:17:42,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742051_1227 (size=12001) 2024-11-12T14:17:42,215 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:42,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:42,221 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/0ebded243c2d45d0a580f037a8e0b9c5 2024-11-12T14:17:42,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/c2ca0410932943109c04eb0d79db248e is 50, key is test_row_0/B:col10/1731421061139/Put/seqid=0 2024-11-12T14:17:42,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742052_1228 (size=12001) 2024-11-12T14:17:42,285 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/c2ca0410932943109c04eb0d79db248e 2024-11-12T14:17:42,285 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:42,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421122276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:42,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:42,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:42,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:42,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:42,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421122329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:42,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:42,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:42,337 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:42,337 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:42,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421122329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:42,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:42,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:42,340 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:42,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/3de2f394235e49f38c731a53c7a7fc50 is 50, key is test_row_0/C:col10/1731421061139/Put/seqid=0 2024-11-12T14:17:42,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:42,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421122390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:42,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:42,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:42,393 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:42,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742053_1229 (size=12001) 2024-11-12T14:17:42,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:42,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421122399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:42,407 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_1' in 7a67a89740591358c1149d6b779d0225 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:17:42,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:42,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:42,407 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:42,417 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:42,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421122410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:42,426 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_0' in 7a67a89740591358c1149d6b779d0225 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:17:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:42,427 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:42,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:42,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421122603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:42,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:42,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:42,610 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:42,673 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T14:17:42,799 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=125 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/3de2f394235e49f38c731a53c7a7fc50 2024-11-12T14:17:42,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/0ebded243c2d45d0a580f037a8e0b9c5 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/0ebded243c2d45d0a580f037a8e0b9c5 2024-11-12T14:17:42,862 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/0ebded243c2d45d0a580f037a8e0b9c5, entries=150, sequenceid=125, filesize=11.7 K 2024-11-12T14:17:42,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/c2ca0410932943109c04eb0d79db248e as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/c2ca0410932943109c04eb0d79db248e 2024-11-12T14:17:42,879 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/c2ca0410932943109c04eb0d79db248e, entries=150, sequenceid=125, filesize=11.7 K 2024-11-12T14:17:42,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/3de2f394235e49f38c731a53c7a7fc50 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/3de2f394235e49f38c731a53c7a7fc50 2024-11-12T14:17:42,890 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/3de2f394235e49f38c731a53c7a7fc50, entries=150, sequenceid=125, filesize=11.7 K 2024-11-12T14:17:42,896 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 7a67a89740591358c1149d6b779d0225 in 759ms, sequenceid=125, compaction requested=true 2024-11-12T14:17:42,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:42,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:42,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-12T14:17:42,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-12T14:17:42,904 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-12T14:17:42,904 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8560 sec 2024-11-12T14:17:42,905 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.8650 sec 2024-11-12T14:17:42,924 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7a67a89740591358c1149d6b779d0225 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T14:17:42,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=A 2024-11-12T14:17:42,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:42,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=B 2024-11-12T14:17:42,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:42,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=C 2024-11-12T14:17:42,924 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:42,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:42,958 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/db2e6faf2ea74eff88a59dc7b07a1d98 is 50, key is test_row_0/A:col10/1731421062274/Put/seqid=0 2024-11-12T14:17:42,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742054_1230 (size=12151) 2024-11-12T14:17:42,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/db2e6faf2ea74eff88a59dc7b07a1d98 2024-11-12T14:17:43,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/20883ef18bee422abfadcfdfff4ba709 is 50, key is test_row_0/B:col10/1731421062274/Put/seqid=0 2024-11-12T14:17:43,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742055_1231 (size=12151) 2024-11-12T14:17:43,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731421063026, max timestamp=1731421063062Num uniques 859; 2024-11-12T14:17:43,071 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T14:17:43,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731421063026, max timestamp=1731421063062Num uniques 859; 2024-11-12T14:17:43,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731421063026, max timestamp=1731421063062Num uniques 859; 2024-11-12T14:17:43,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T14:17:43,074 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T14:17:43,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:43,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421123205, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:43,214 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:43,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:43,215 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:43,326 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:43,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421123319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:43,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:43,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:43,326 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:43,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:43,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421123350, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:43,356 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_2' in 7a67a89740591358c1149d6b779d0225 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:17:43,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:43,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:43,357 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:43,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:43,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421123360, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:43,366 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_2' in 7a67a89740591358c1149d6b779d0225 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:17:43,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:43,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:43,367 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:43,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/20883ef18bee422abfadcfdfff4ba709 2024-11-12T14:17:43,459 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/519b173fcf234caabd2e8aeb8cb8616e is 50, key is test_row_0/C:col10/1731421062274/Put/seqid=0 2024-11-12T14:17:43,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742056_1232 (size=12151) 2024-11-12T14:17:43,489 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/519b173fcf234caabd2e8aeb8cb8616e 2024-11-12T14:17:43,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/db2e6faf2ea74eff88a59dc7b07a1d98 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/db2e6faf2ea74eff88a59dc7b07a1d98 2024-11-12T14:17:43,513 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/db2e6faf2ea74eff88a59dc7b07a1d98, entries=150, sequenceid=136, filesize=11.9 K 2024-11-12T14:17:43,515 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/20883ef18bee422abfadcfdfff4ba709 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/20883ef18bee422abfadcfdfff4ba709 2024-11-12T14:17:43,521 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/20883ef18bee422abfadcfdfff4ba709, entries=150, sequenceid=136, filesize=11.9 K 2024-11-12T14:17:43,522 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/519b173fcf234caabd2e8aeb8cb8616e as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/519b173fcf234caabd2e8aeb8cb8616e 2024-11-12T14:17:43,528 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/519b173fcf234caabd2e8aeb8cb8616e, entries=150, sequenceid=136, filesize=11.9 K 2024-11-12T14:17:43,530 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 7a67a89740591358c1149d6b779d0225 in 606ms, sequenceid=136, compaction requested=true 2024-11-12T14:17:43,530 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:43,530 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:17:43,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a67a89740591358c1149d6b779d0225:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:17:43,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:43,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a67a89740591358c1149d6b779d0225:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:17:43,530 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:43,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a67a89740591358c1149d6b779d0225:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:17:43,531 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T14:17:43,531 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:17:43,532 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50494 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:17:43,532 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 7a67a89740591358c1149d6b779d0225/A is initiating minor compaction (all files) 2024-11-12T14:17:43,532 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a67a89740591358c1149d6b779d0225/A in TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:43,532 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/35ae6d1dee954d0d9ac630030115b152, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/b00ff763a1cb499eb2548ffbe473a361, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/0ebded243c2d45d0a580f037a8e0b9c5, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/db2e6faf2ea74eff88a59dc7b07a1d98] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp, totalSize=49.3 K 2024-11-12T14:17:43,533 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48154 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:17:43,533 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 7a67a89740591358c1149d6b779d0225/B is initiating minor compaction (all files) 2024-11-12T14:17:43,533 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a67a89740591358c1149d6b779d0225/B in TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:43,533 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/69946287a9eb4ddaada2e2959c0a7774, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/c15c09973b9246ccbd052d5cf9314784, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/c2ca0410932943109c04eb0d79db248e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/20883ef18bee422abfadcfdfff4ba709] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp, totalSize=47.0 K 2024-11-12T14:17:43,533 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 35ae6d1dee954d0d9ac630030115b152, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731421060050 2024-11-12T14:17:43,533 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69946287a9eb4ddaada2e2959c0a7774, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731421060050 2024-11-12T14:17:43,533 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting b00ff763a1cb499eb2548ffbe473a361, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731421060290 2024-11-12T14:17:43,533 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting c15c09973b9246ccbd052d5cf9314784, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731421060290 2024-11-12T14:17:43,534 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ebded243c2d45d0a580f037a8e0b9c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731421061123 2024-11-12T14:17:43,534 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2ca0410932943109c04eb0d79db248e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731421061123 2024-11-12T14:17:43,534 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting db2e6faf2ea74eff88a59dc7b07a1d98, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731421062242 2024-11-12T14:17:43,535 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20883ef18bee422abfadcfdfff4ba709, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731421062242 2024-11-12T14:17:43,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7a67a89740591358c1149d6b779d0225 3/3 column families, dataSize=134.18 KB heapSize=384.98 KB 2024-11-12T14:17:43,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=A 2024-11-12T14:17:43,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:17:43,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=B 2024-11-12T14:17:43,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:17:43,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=C 2024-11-12T14:17:43,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:17:43,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:43,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/afa5a740bbbc4e4a9f81397e53e9fd55 is 50, key is test_row_0/A:col10/1731421063536/Put/seqid=0 2024-11-12T14:17:43,576 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a67a89740591358c1149d6b779d0225#A#compaction#188 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:43,576 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a67a89740591358c1149d6b779d0225#B#compaction#187 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:43,577 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/10c9f612a8e24d1aa5df715b501ad97c is 50, key is test_row_0/B:col10/1731421062274/Put/seqid=0 2024-11-12T14:17:43,578 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/629954e0385c4cd4a045fc3141d31f8a is 50, key is test_row_0/A:col10/1731421062274/Put/seqid=0 2024-11-12T14:17:43,596 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742058_1234 (size=12289) 2024-11-12T14:17:43,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742057_1233 (size=14541) 2024-11-12T14:17:43,617 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:43,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421123613, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:43,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:43,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:43,618 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:43,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742059_1235 (size=12289) 2024-11-12T14:17:43,651 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/629954e0385c4cd4a045fc3141d31f8a as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/629954e0385c4cd4a045fc3141d31f8a 2024-11-12T14:17:43,662 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7a67a89740591358c1149d6b779d0225/A of 7a67a89740591358c1149d6b779d0225 into 629954e0385c4cd4a045fc3141d31f8a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:43,662 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:43,662 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., storeName=7a67a89740591358c1149d6b779d0225/A, priority=12, startTime=1731421063530; duration=0sec 2024-11-12T14:17:43,662 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:43,662 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a67a89740591358c1149d6b779d0225:A 2024-11-12T14:17:43,662 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:17:43,664 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48154 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:17:43,664 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 7a67a89740591358c1149d6b779d0225/C is initiating minor compaction (all files) 2024-11-12T14:17:43,664 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a67a89740591358c1149d6b779d0225/C in TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:43,664 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/32d725af0bbb4b25ae8b71e7021d07af, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/1cb3e579dc114cc0bf97244f38a391aa, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/3de2f394235e49f38c731a53c7a7fc50, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/519b173fcf234caabd2e8aeb8cb8616e] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp, totalSize=47.0 K 2024-11-12T14:17:43,665 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 32d725af0bbb4b25ae8b71e7021d07af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=11, earliestPutTs=1731421060050 2024-11-12T14:17:43,665 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 1cb3e579dc114cc0bf97244f38a391aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731421060290 2024-11-12T14:17:43,665 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 3de2f394235e49f38c731a53c7a7fc50, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=125, earliestPutTs=1731421061123 2024-11-12T14:17:43,666 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 519b173fcf234caabd2e8aeb8cb8616e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731421062242 2024-11-12T14:17:43,692 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a67a89740591358c1149d6b779d0225#C#compaction#189 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:43,694 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/546f5920b5ae467483f868ea5bd3d77d is 50, key is test_row_0/C:col10/1731421062274/Put/seqid=0 2024-11-12T14:17:43,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742060_1236 (size=12289) 2024-11-12T14:17:43,734 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421123730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:43,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:43,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:43,734 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:43,946 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:43,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421123939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:43,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:43,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:43,947 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:44,002 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/10c9f612a8e24d1aa5df715b501ad97c as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/10c9f612a8e24d1aa5df715b501ad97c 2024-11-12T14:17:44,009 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/afa5a740bbbc4e4a9f81397e53e9fd55 2024-11-12T14:17:44,013 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7a67a89740591358c1149d6b779d0225/B of 7a67a89740591358c1149d6b779d0225 into 10c9f612a8e24d1aa5df715b501ad97c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:44,013 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:44,013 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., storeName=7a67a89740591358c1149d6b779d0225/B, priority=12, startTime=1731421063530; duration=0sec 2024-11-12T14:17:44,013 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:44,013 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a67a89740591358c1149d6b779d0225:B 2024-11-12T14:17:44,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/6c78d05c25194b2b9ca6032eba87ca74 is 50, key is test_row_0/B:col10/1731421063536/Put/seqid=0 2024-11-12T14:17:44,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742061_1237 (size=12151) 2024-11-12T14:17:44,164 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/546f5920b5ae467483f868ea5bd3d77d as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/546f5920b5ae467483f868ea5bd3d77d 2024-11-12T14:17:44,189 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7a67a89740591358c1149d6b779d0225/C of 7a67a89740591358c1149d6b779d0225 into 546f5920b5ae467483f868ea5bd3d77d(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:44,189 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:44,190 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., storeName=7a67a89740591358c1149d6b779d0225/C, priority=12, startTime=1731421063530; duration=0sec 2024-11-12T14:17:44,190 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:44,190 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a67a89740591358c1149d6b779d0225:C 2024-11-12T14:17:44,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-12T14:17:44,219 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:17:44,220 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:17:44,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-12T14:17:44,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T14:17:44,225 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:17:44,226 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:17:44,226 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:17:44,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:44,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421124260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:44,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:44,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:44,260 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:44,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T14:17:44,378 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-12T14:17:44,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:44,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:44,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:44,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:44,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:44,433 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:44,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421124429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:44,433 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_1' in 7a67a89740591358c1149d6b779d0225 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:17:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:44,434 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:44,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:44,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421124451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:44,460 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/6c78d05c25194b2b9ca6032eba87ca74 2024-11-12T14:17:44,460 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_0' in 7a67a89740591358c1149d6b779d0225 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:17:44,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:44,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:44,461 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:44,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/e4b5b684dc8e4d799e963c14e1afdc67 is 50, key is test_row_0/C:col10/1731421063536/Put/seqid=0 2024-11-12T14:17:44,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742062_1238 (size=12151) 2024-11-12T14:17:44,531 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-12T14:17:44,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:44,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:44,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:44,532 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:44,532 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:44,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:44,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T14:17:44,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-12T14:17:44,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:44,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:44,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:44,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:44,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:44,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:44,780 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:44,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421124776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:44,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:44,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:44,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:44,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T14:17:44,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-12T14:17:44,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:44,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:44,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:44,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:44,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:44,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:44,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/e4b5b684dc8e4d799e963c14e1afdc67 2024-11-12T14:17:44,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/afa5a740bbbc4e4a9f81397e53e9fd55 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/afa5a740bbbc4e4a9f81397e53e9fd55 2024-11-12T14:17:44,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/afa5a740bbbc4e4a9f81397e53e9fd55, entries=200, sequenceid=173, filesize=14.2 K 2024-11-12T14:17:44,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/6c78d05c25194b2b9ca6032eba87ca74 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/6c78d05c25194b2b9ca6032eba87ca74 2024-11-12T14:17:45,002 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/6c78d05c25194b2b9ca6032eba87ca74, entries=150, sequenceid=173, filesize=11.9 K 2024-11-12T14:17:45,012 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/e4b5b684dc8e4d799e963c14e1afdc67 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/e4b5b684dc8e4d799e963c14e1afdc67 2024-11-12T14:17:45,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-12T14:17:45,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:45,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:45,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:45,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:45,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:45,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:45,038 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/e4b5b684dc8e4d799e963c14e1afdc67, entries=150, sequenceid=173, filesize=11.9 K 2024-11-12T14:17:45,041 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~384.94 KB/394176, currentSize=46.96 KB/48090 for 7a67a89740591358c1149d6b779d0225 in 1502ms, sequenceid=173, compaction requested=false 2024-11-12T14:17:45,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:45,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-12T14:17:45,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:45,181 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 7a67a89740591358c1149d6b779d0225 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T14:17:45,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=A 2024-11-12T14:17:45,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:45,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=B 2024-11-12T14:17:45,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:45,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=C 2024-11-12T14:17:45,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:45,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/a897c7ed81cc451c9a4929ecc527cbf1 is 50, key is test_row_0/A:col10/1731421063569/Put/seqid=0 2024-11-12T14:17:45,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742063_1239 (size=12151) 2024-11-12T14:17:45,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T14:17:45,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:45,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:45,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731421065425, max timestamp=1731421065442Num uniques 859; 2024-11-12T14:17:45,445 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T14:17:45,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731421065425, max timestamp=1731421065444Num uniques 859; 2024-11-12T14:17:45,447 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T14:17:45,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731421065425, max timestamp=1731421065442Num uniques 859; 2024-11-12T14:17:45,448 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T14:17:45,507 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:45,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421125500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:45,507 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:45,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:45,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:45,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:45,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421125502, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:45,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:45,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:45,511 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:45,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:45,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421125619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:45,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:45,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:45,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421125620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:45,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:45,624 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:45,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:45,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:45,624 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:45,640 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/a897c7ed81cc451c9a4929ecc527cbf1 2024-11-12T14:17:45,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/ee0758775ce04336b1097822c452540e is 50, key is test_row_0/B:col10/1731421063569/Put/seqid=0 2024-11-12T14:17:45,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742064_1240 (size=12151) 2024-11-12T14:17:45,736 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/ee0758775ce04336b1097822c452540e 2024-11-12T14:17:45,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/c2632106b0fa4a5a98a4b8ff5ee4fd21 is 50, key is test_row_0/C:col10/1731421063569/Put/seqid=0 2024-11-12T14:17:45,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:45,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421125792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:45,800 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_0' in 7a67a89740591358c1149d6b779d0225 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:17:45,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:45,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:45,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:45,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742065_1241 (size=12151) 2024-11-12T14:17:45,813 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=186 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/c2632106b0fa4a5a98a4b8ff5ee4fd21 2024-11-12T14:17:45,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/a897c7ed81cc451c9a4929ecc527cbf1 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/a897c7ed81cc451c9a4929ecc527cbf1 2024-11-12T14:17:45,830 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/a897c7ed81cc451c9a4929ecc527cbf1, entries=150, sequenceid=186, filesize=11.9 K 2024-11-12T14:17:45,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/ee0758775ce04336b1097822c452540e as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/ee0758775ce04336b1097822c452540e 2024-11-12T14:17:45,839 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:45,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421125830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:45,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:45,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:45,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:45,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421125830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:45,840 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:45,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:45,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:45,840 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:45,841 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/ee0758775ce04336b1097822c452540e, entries=150, sequenceid=186, filesize=11.9 K 2024-11-12T14:17:45,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/c2632106b0fa4a5a98a4b8ff5ee4fd21 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/c2632106b0fa4a5a98a4b8ff5ee4fd21 2024-11-12T14:17:45,849 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/c2632106b0fa4a5a98a4b8ff5ee4fd21, entries=150, sequenceid=186, filesize=11.9 K 2024-11-12T14:17:45,852 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 7a67a89740591358c1149d6b779d0225 in 672ms, sequenceid=186, compaction requested=true 2024-11-12T14:17:45,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:45,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:45,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-12T14:17:45,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-12T14:17:45,855 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-12T14:17:45,856 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6280 sec 2024-11-12T14:17:45,858 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.6360 sec 2024-11-12T14:17:46,155 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7a67a89740591358c1149d6b779d0225 3/3 column families, dataSize=140.89 KB heapSize=405.61 KB 2024-11-12T14:17:46,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=A 2024-11-12T14:17:46,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T14:17:46,155 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=B 2024-11-12T14:17:46,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:46,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731421065483, max timestamp=1731421066151Num uniques 1018; 2024-11-12T14:17:46,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46618, new segment data size=7688, suffix heap size=133840,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1018, new segment cells count=168 2024-11-12T14:17:46,156 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=3, new segment=null 2024-11-12T14:17:46,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=C 2024-11-12T14:17:46,160 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:17:46,174 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/3bd71df898a043f9bc0ae43c20e9bb86 is 50, key is test_row_0/A:col10/1731421065500/Put/seqid=0 2024-11-12T14:17:46,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:46,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421126201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:46,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:46,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421126202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:46,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:46,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:46,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:46,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:46,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:46,208 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:46,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742066_1242 (size=12151) 2024-11-12T14:17:46,211 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/3bd71df898a043f9bc0ae43c20e9bb86 2024-11-12T14:17:46,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/21bdfdfbef9b41a9bfc220897b7fc44c is 50, key is test_row_0/B:col10/1731421065500/Put/seqid=0 2024-11-12T14:17:46,276 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742067_1243 (size=12151) 2024-11-12T14:17:46,315 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:46,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421126309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:46,315 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:46,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:46,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421126310, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:46,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:46,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:46,316 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:46,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-12T14:17:46,372 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:17:46,373 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:17:46,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-12T14:17:46,375 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:17:46,376 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:17:46,376 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:17:46,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T14:17:46,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T14:17:46,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:46,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421126519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:46,522 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:46,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421126520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:46,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:46,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:46,522 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:46,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:46,530 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-12T14:17:46,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:46,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:46,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:46,530 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:46,530 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:46,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:46,679 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/21bdfdfbef9b41a9bfc220897b7fc44c 2024-11-12T14:17:46,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-12T14:17:46,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:46,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:46,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:46,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:46,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:46,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:46,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T14:17:46,721 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/9893efef67204581a5427a998c71b94d is 50, key is test_row_0/C:col10/1731421065500/Put/seqid=0 2024-11-12T14:17:46,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742068_1244 (size=12151) 2024-11-12T14:17:46,753 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/9893efef67204581a5427a998c71b94d 2024-11-12T14:17:46,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/3bd71df898a043f9bc0ae43c20e9bb86 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/3bd71df898a043f9bc0ae43c20e9bb86 2024-11-12T14:17:46,779 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/3bd71df898a043f9bc0ae43c20e9bb86, entries=150, sequenceid=224, filesize=11.9 K 2024-11-12T14:17:46,780 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/21bdfdfbef9b41a9bfc220897b7fc44c as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/21bdfdfbef9b41a9bfc220897b7fc44c 2024-11-12T14:17:46,794 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/21bdfdfbef9b41a9bfc220897b7fc44c, entries=150, sequenceid=224, filesize=11.9 K 2024-11-12T14:17:46,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/9893efef67204581a5427a998c71b94d as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/9893efef67204581a5427a998c71b94d 2024-11-12T14:17:46,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/9893efef67204581a5427a998c71b94d, entries=150, sequenceid=224, filesize=11.9 K 2024-11-12T14:17:46,820 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~102.87 KB/105340, heapSize ~292.16 KB/299168, currentSize=73.80 KB/75570 for 7a67a89740591358c1149d6b779d0225 in 665ms, sequenceid=224, compaction requested=true 2024-11-12T14:17:46,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:46,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a67a89740591358c1149d6b779d0225:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:17:46,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:46,821 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:17:46,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a67a89740591358c1149d6b779d0225:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:17:46,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:46,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a67a89740591358c1149d6b779d0225:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:17:46,821 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T14:17:46,821 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:17:46,830 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48742 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:17:46,830 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 7a67a89740591358c1149d6b779d0225/B is initiating minor compaction (all files) 2024-11-12T14:17:46,830 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a67a89740591358c1149d6b779d0225/B in TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:46,830 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/10c9f612a8e24d1aa5df715b501ad97c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/6c78d05c25194b2b9ca6032eba87ca74, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/ee0758775ce04336b1097822c452540e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/21bdfdfbef9b41a9bfc220897b7fc44c] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp, totalSize=47.6 K 2024-11-12T14:17:46,831 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51132 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:17:46,831 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 7a67a89740591358c1149d6b779d0225/A is initiating minor compaction (all files) 2024-11-12T14:17:46,831 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a67a89740591358c1149d6b779d0225/A in TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:46,831 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/629954e0385c4cd4a045fc3141d31f8a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/afa5a740bbbc4e4a9f81397e53e9fd55, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/a897c7ed81cc451c9a4929ecc527cbf1, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/3bd71df898a043f9bc0ae43c20e9bb86] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp, totalSize=49.9 K 2024-11-12T14:17:46,831 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 10c9f612a8e24d1aa5df715b501ad97c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731421062242 2024-11-12T14:17:46,831 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 629954e0385c4cd4a045fc3141d31f8a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731421062242 2024-11-12T14:17:46,832 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c78d05c25194b2b9ca6032eba87ca74, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731421063182 2024-11-12T14:17:46,832 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting afa5a740bbbc4e4a9f81397e53e9fd55, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731421063182 2024-11-12T14:17:46,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:46,833 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7a67a89740591358c1149d6b779d0225 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T14:17:46,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=A 2024-11-12T14:17:46,833 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting a897c7ed81cc451c9a4929ecc527cbf1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731421063569 2024-11-12T14:17:46,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:46,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=B 2024-11-12T14:17:46,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:46,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=C 2024-11-12T14:17:46,834 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting ee0758775ce04336b1097822c452540e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731421063569 2024-11-12T14:17:46,834 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:46,834 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bd71df898a043f9bc0ae43c20e9bb86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731421065498 2024-11-12T14:17:46,834 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 21bdfdfbef9b41a9bfc220897b7fc44c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731421065498 2024-11-12T14:17:46,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-12T14:17:46,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:46,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:46,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:46,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:46,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:46,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:46,839 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/c234cc08fe9c44eabd9ceca3de03dcf9 is 50, key is test_row_0/A:col10/1731421066830/Put/seqid=0 2024-11-12T14:17:46,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742069_1245 (size=14541) 2024-11-12T14:17:46,849 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a67a89740591358c1149d6b779d0225#B#compaction#199 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:46,850 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/0e7f564e0224424ab7eb1338a5c7e814 is 50, key is test_row_0/B:col10/1731421065500/Put/seqid=0 2024-11-12T14:17:46,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/c234cc08fe9c44eabd9ceca3de03dcf9 2024-11-12T14:17:46,865 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a67a89740591358c1149d6b779d0225#A#compaction#200 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:46,866 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/1809e3f6feab46408316b04e0e2c65a7 is 50, key is test_row_0/A:col10/1731421065500/Put/seqid=0 2024-11-12T14:17:46,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742070_1246 (size=12425) 2024-11-12T14:17:46,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742071_1247 (size=12425) 2024-11-12T14:17:46,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/f1d73a37294c452bb6f9923149f03c90 is 50, key is test_row_0/B:col10/1731421066830/Put/seqid=0 2024-11-12T14:17:46,901 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:46,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421126893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:46,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:46,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:46,902 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:46,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:46,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421126902, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:46,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:46,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:46,910 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:46,928 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/1809e3f6feab46408316b04e0e2c65a7 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/1809e3f6feab46408316b04e0e2c65a7 2024-11-12T14:17:46,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742072_1248 (size=12151) 2024-11-12T14:17:46,947 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/f1d73a37294c452bb6f9923149f03c90 2024-11-12T14:17:46,953 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7a67a89740591358c1149d6b779d0225/A of 7a67a89740591358c1149d6b779d0225 into 1809e3f6feab46408316b04e0e2c65a7(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:46,953 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:46,953 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., storeName=7a67a89740591358c1149d6b779d0225/A, priority=12, startTime=1731421066821; duration=0sec 2024-11-12T14:17:46,953 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:46,953 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a67a89740591358c1149d6b779d0225:A 2024-11-12T14:17:46,953 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:17:46,959 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48742 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:17:46,959 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 7a67a89740591358c1149d6b779d0225/C is initiating minor compaction (all files) 2024-11-12T14:17:46,959 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a67a89740591358c1149d6b779d0225/C in TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:46,959 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/546f5920b5ae467483f868ea5bd3d77d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/e4b5b684dc8e4d799e963c14e1afdc67, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/c2632106b0fa4a5a98a4b8ff5ee4fd21, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/9893efef67204581a5427a998c71b94d] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp, totalSize=47.6 K 2024-11-12T14:17:46,960 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 546f5920b5ae467483f868ea5bd3d77d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1731421062242 2024-11-12T14:17:46,962 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4b5b684dc8e4d799e963c14e1afdc67, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1731421063182 2024-11-12T14:17:46,962 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2632106b0fa4a5a98a4b8ff5ee4fd21, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=186, earliestPutTs=1731421063569 2024-11-12T14:17:46,962 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9893efef67204581a5427a998c71b94d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731421065498 2024-11-12T14:17:46,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/9e432300f0d24aec9e658f0cff422b0d is 50, key is test_row_0/C:col10/1731421066830/Put/seqid=0 2024-11-12T14:17:46,982 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a67a89740591358c1149d6b779d0225#C#compaction#203 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:46,983 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/c87fa5ce56ee415c8e6d3995b3b80f21 is 50, key is test_row_0/C:col10/1731421065500/Put/seqid=0 2024-11-12T14:17:47,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-12T14:17:47,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:47,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:47,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:47,002 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:47,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:47,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:47,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T14:17:47,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:47,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421127009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:47,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:47,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:47,014 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:47,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742073_1249 (size=12151) 2024-11-12T14:17:47,023 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:47,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421127019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:47,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:47,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:47,024 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:47,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742074_1250 (size=12425) 2024-11-12T14:17:47,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-12T14:17:47,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:47,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:47,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:47,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:47,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:47,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:47,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:47,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421127219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:47,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:47,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:47,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:47,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:47,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421127229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:47,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:47,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:47,231 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:47,292 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/0e7f564e0224424ab7eb1338a5c7e814 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/0e7f564e0224424ab7eb1338a5c7e814 2024-11-12T14:17:47,305 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7a67a89740591358c1149d6b779d0225/B of 7a67a89740591358c1149d6b779d0225 into 0e7f564e0224424ab7eb1338a5c7e814(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:47,305 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:47,306 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., storeName=7a67a89740591358c1149d6b779d0225/B, priority=12, startTime=1731421066821; duration=0sec 2024-11-12T14:17:47,306 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:47,306 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a67a89740591358c1149d6b779d0225:B 2024-11-12T14:17:47,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-12T14:17:47,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:47,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:47,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:47,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:47,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:47,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:47,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/9e432300f0d24aec9e658f0cff422b0d 2024-11-12T14:17:47,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/c234cc08fe9c44eabd9ceca3de03dcf9 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/c234cc08fe9c44eabd9ceca3de03dcf9 2024-11-12T14:17:47,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/c234cc08fe9c44eabd9ceca3de03dcf9, entries=200, sequenceid=239, filesize=14.2 K 2024-11-12T14:17:47,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/f1d73a37294c452bb6f9923149f03c90 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/f1d73a37294c452bb6f9923149f03c90 2024-11-12T14:17:47,452 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/c87fa5ce56ee415c8e6d3995b3b80f21 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/c87fa5ce56ee415c8e6d3995b3b80f21 2024-11-12T14:17:47,458 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/f1d73a37294c452bb6f9923149f03c90, entries=150, sequenceid=239, filesize=11.9 K 2024-11-12T14:17:47,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/9e432300f0d24aec9e658f0cff422b0d as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/9e432300f0d24aec9e658f0cff422b0d 2024-11-12T14:17:47,462 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7a67a89740591358c1149d6b779d0225/C of 7a67a89740591358c1149d6b779d0225 into c87fa5ce56ee415c8e6d3995b3b80f21(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:47,462 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:47,462 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., storeName=7a67a89740591358c1149d6b779d0225/C, priority=12, startTime=1731421066821; duration=0sec 2024-11-12T14:17:47,462 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:47,462 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a67a89740591358c1149d6b779d0225:C 2024-11-12T14:17:47,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/9e432300f0d24aec9e658f0cff422b0d, entries=150, sequenceid=239, filesize=11.9 K 2024-11-12T14:17:47,479 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 7a67a89740591358c1149d6b779d0225 in 646ms, sequenceid=239, compaction requested=false 2024-11-12T14:17:47,479 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:47,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-12T14:17:47,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:47,484 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 7a67a89740591358c1149d6b779d0225 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T14:17:47,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=A 2024-11-12T14:17:47,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:47,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=B 2024-11-12T14:17:47,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:47,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=C 2024-11-12T14:17:47,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:47,504 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/be3432373b02415388b52efcd8ee16d8 is 50, key is test_row_0/A:col10/1731421066859/Put/seqid=0 2024-11-12T14:17:47,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T14:17:47,532 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742075_1251 (size=12151) 2024-11-12T14:17:47,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:47,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:47,537 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/be3432373b02415388b52efcd8ee16d8 2024-11-12T14:17:47,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/05ab3bfba2e04a85be974a4106e7669c is 50, key is test_row_0/B:col10/1731421066859/Put/seqid=0 2024-11-12T14:17:47,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742076_1252 (size=12151) 2024-11-12T14:17:47,586 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:47,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421127580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:47,590 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:47,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421127583, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:47,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:47,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:47,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:47,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:47,592 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:47,593 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:47,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:47,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421127699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:47,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:47,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421127700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:47,702 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:47,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:47,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:47,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:47,821 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:47,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421127819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:47,821 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_0' in 7a67a89740591358c1149d6b779d0225 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4208 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:17:47,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:47,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:47,822 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:47,913 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:47,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421127910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:47,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:47,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421127912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:47,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:47,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:47,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:47,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:47,916 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:47,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:47,976 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/05ab3bfba2e04a85be974a4106e7669c 2024-11-12T14:17:47,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/bb68c640b73544b9813e38c582c75f5f is 50, key is test_row_0/C:col10/1731421066859/Put/seqid=0 2024-11-12T14:17:47,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742077_1253 (size=12151) 2024-11-12T14:17:48,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:48,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421128219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:48,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:48,224 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:48,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:48,224 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:48,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421128220, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:48,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:48,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:48,224 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:48,388 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=259 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/bb68c640b73544b9813e38c582c75f5f 2024-11-12T14:17:48,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/be3432373b02415388b52efcd8ee16d8 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/be3432373b02415388b52efcd8ee16d8 2024-11-12T14:17:48,395 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/be3432373b02415388b52efcd8ee16d8, entries=150, sequenceid=259, filesize=11.9 K 2024-11-12T14:17:48,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/05ab3bfba2e04a85be974a4106e7669c as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/05ab3bfba2e04a85be974a4106e7669c 2024-11-12T14:17:48,400 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/05ab3bfba2e04a85be974a4106e7669c, entries=150, sequenceid=259, filesize=11.9 K 2024-11-12T14:17:48,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/bb68c640b73544b9813e38c582c75f5f as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/bb68c640b73544b9813e38c582c75f5f 2024-11-12T14:17:48,404 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/bb68c640b73544b9813e38c582c75f5f, entries=150, sequenceid=259, filesize=11.9 K 2024-11-12T14:17:48,405 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 7a67a89740591358c1149d6b779d0225 in 921ms, sequenceid=259, compaction requested=true 2024-11-12T14:17:48,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:48,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:48,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-12T14:17:48,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-12T14:17:48,408 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-12T14:17:48,408 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0310 sec 2024-11-12T14:17:48,410 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 2.0360 sec 2024-11-12T14:17:48,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:48,452 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7a67a89740591358c1149d6b779d0225 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-12T14:17:48,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=A 2024-11-12T14:17:48,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:48,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=B 2024-11-12T14:17:48,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:48,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=C 2024-11-12T14:17:48,452 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:48,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/0052da1763114897a7dfad3db15c2bde is 50, key is test_row_0/A:col10/1731421067580/Put/seqid=0 2024-11-12T14:17:48,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742078_1254 (size=14741) 2024-11-12T14:17:48,465 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/0052da1763114897a7dfad3db15c2bde 2024-11-12T14:17:48,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/3ee3aa87f7a24254ab586675730ac11d is 50, key is test_row_0/B:col10/1731421067580/Put/seqid=0 2024-11-12T14:17:48,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742079_1255 (size=12301) 2024-11-12T14:17:48,480 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/3ee3aa87f7a24254ab586675730ac11d 2024-11-12T14:17:48,488 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/0b652acf7e2544f887f4d08d2bf43808 is 50, key is test_row_0/C:col10/1731421067580/Put/seqid=0 2024-11-12T14:17:48,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742080_1256 (size=12301) 2024-11-12T14:17:48,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:48,519 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:48,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421128512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:48,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421128513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:48,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:48,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:48,520 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:48,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:48,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:48,520 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:48,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-12T14:17:48,540 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:17:48,540 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:17:48,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-12T14:17:48,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-12T14:17:48,543 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:17:48,543 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:17:48,543 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:17:48,632 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:48,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421128629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:48,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:48,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:48,632 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:48,634 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:48,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421128630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:48,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:48,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:48,635 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:48,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-12T14:17:48,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-12T14:17:48,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:48,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:48,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:48,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:48,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:48,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:48,732 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:48,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421128729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:48,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:48,733 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:48,742 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:48,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421128739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:48,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:48,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:48,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:48,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:48,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421128839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:48,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:48,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:48,843 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:48,843 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:48,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421128839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:48,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:48,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:48,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:48,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-12T14:17:48,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:48,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:48,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:48,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] handler.RSProcedureHandler(58): pid=59 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:48,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=59 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:48,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=59 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:48,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-12T14:17:48,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/0b652acf7e2544f887f4d08d2bf43808 2024-11-12T14:17:48,898 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/0052da1763114897a7dfad3db15c2bde as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/0052da1763114897a7dfad3db15c2bde 2024-11-12T14:17:48,901 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/0052da1763114897a7dfad3db15c2bde, entries=200, sequenceid=275, filesize=14.4 K 2024-11-12T14:17:48,902 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/3ee3aa87f7a24254ab586675730ac11d as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/3ee3aa87f7a24254ab586675730ac11d 2024-11-12T14:17:48,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/3ee3aa87f7a24254ab586675730ac11d, entries=150, sequenceid=275, filesize=12.0 K 2024-11-12T14:17:48,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/0b652acf7e2544f887f4d08d2bf43808 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/0b652acf7e2544f887f4d08d2bf43808 2024-11-12T14:17:48,910 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/0b652acf7e2544f887f4d08d2bf43808, entries=150, sequenceid=275, filesize=12.0 K 2024-11-12T14:17:48,911 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 7a67a89740591358c1149d6b779d0225 in 459ms, sequenceid=275, compaction requested=true 2024-11-12T14:17:48,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:48,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a67a89740591358c1149d6b779d0225:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:17:48,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:48,911 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:17:48,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a67a89740591358c1149d6b779d0225:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:17:48,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:48,911 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:17:48,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a67a89740591358c1149d6b779d0225:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:17:48,911 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:48,912 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53858 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:17:48,912 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49028 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:17:48,912 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 7a67a89740591358c1149d6b779d0225/A is initiating minor compaction (all files) 2024-11-12T14:17:48,912 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 7a67a89740591358c1149d6b779d0225/B is initiating minor compaction (all files) 2024-11-12T14:17:48,912 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a67a89740591358c1149d6b779d0225/A in TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:48,912 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a67a89740591358c1149d6b779d0225/B in TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:48,912 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/0e7f564e0224424ab7eb1338a5c7e814, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/f1d73a37294c452bb6f9923149f03c90, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/05ab3bfba2e04a85be974a4106e7669c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/3ee3aa87f7a24254ab586675730ac11d] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp, totalSize=47.9 K 2024-11-12T14:17:48,912 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/1809e3f6feab46408316b04e0e2c65a7, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/c234cc08fe9c44eabd9ceca3de03dcf9, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/be3432373b02415388b52efcd8ee16d8, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/0052da1763114897a7dfad3db15c2bde] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp, totalSize=52.6 K 2024-11-12T14:17:48,913 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1809e3f6feab46408316b04e0e2c65a7, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731421065498 2024-11-12T14:17:48,913 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e7f564e0224424ab7eb1338a5c7e814, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731421065498 2024-11-12T14:17:48,913 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting c234cc08fe9c44eabd9ceca3de03dcf9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731421066166 2024-11-12T14:17:48,913 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting f1d73a37294c452bb6f9923149f03c90, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731421066191 2024-11-12T14:17:48,913 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting be3432373b02415388b52efcd8ee16d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731421066859 2024-11-12T14:17:48,913 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 05ab3bfba2e04a85be974a4106e7669c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731421066859 2024-11-12T14:17:48,914 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0052da1763114897a7dfad3db15c2bde, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731421067551 2024-11-12T14:17:48,914 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ee3aa87f7a24254ab586675730ac11d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731421067551 2024-11-12T14:17:48,922 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a67a89740591358c1149d6b779d0225#B#compaction#210 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:48,922 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/59c61fc918354c03810426fae327f65e is 50, key is test_row_0/B:col10/1731421067580/Put/seqid=0 2024-11-12T14:17:48,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742081_1257 (size=12711) 2024-11-12T14:17:48,932 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/59c61fc918354c03810426fae327f65e as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/59c61fc918354c03810426fae327f65e 2024-11-12T14:17:48,932 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a67a89740591358c1149d6b779d0225#A#compaction#211 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:48,933 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/b210b2aeb99a4df59e26d5b13368e972 is 50, key is test_row_0/A:col10/1731421067580/Put/seqid=0 2024-11-12T14:17:48,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742082_1258 (size=12711) 2024-11-12T14:17:48,939 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7a67a89740591358c1149d6b779d0225/B of 7a67a89740591358c1149d6b779d0225 into 59c61fc918354c03810426fae327f65e(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:48,939 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:48,939 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., storeName=7a67a89740591358c1149d6b779d0225/B, priority=12, startTime=1731421068911; duration=0sec 2024-11-12T14:17:48,940 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:48,940 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a67a89740591358c1149d6b779d0225:B 2024-11-12T14:17:48,940 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:17:48,943 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49028 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:17:48,944 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 7a67a89740591358c1149d6b779d0225/C is initiating minor compaction (all files) 2024-11-12T14:17:48,944 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a67a89740591358c1149d6b779d0225/C in TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:48,944 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/c87fa5ce56ee415c8e6d3995b3b80f21, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/9e432300f0d24aec9e658f0cff422b0d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/bb68c640b73544b9813e38c582c75f5f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/0b652acf7e2544f887f4d08d2bf43808] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp, totalSize=47.9 K 2024-11-12T14:17:48,944 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting c87fa5ce56ee415c8e6d3995b3b80f21, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731421065498 2024-11-12T14:17:48,945 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e432300f0d24aec9e658f0cff422b0d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731421066191 2024-11-12T14:17:48,945 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting bb68c640b73544b9813e38c582c75f5f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=259, earliestPutTs=1731421066859 2024-11-12T14:17:48,946 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b652acf7e2544f887f4d08d2bf43808, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731421067551 2024-11-12T14:17:48,946 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/b210b2aeb99a4df59e26d5b13368e972 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/b210b2aeb99a4df59e26d5b13368e972 2024-11-12T14:17:48,952 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7a67a89740591358c1149d6b779d0225/A of 7a67a89740591358c1149d6b779d0225 into b210b2aeb99a4df59e26d5b13368e972(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:48,952 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:48,952 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., storeName=7a67a89740591358c1149d6b779d0225/A, priority=12, startTime=1731421068911; duration=0sec 2024-11-12T14:17:48,952 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:48,952 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a67a89740591358c1149d6b779d0225:A 2024-11-12T14:17:48,957 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a67a89740591358c1149d6b779d0225#C#compaction#212 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:48,957 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/b27a2b9141bc4b4a88beab9abfc62f7a is 50, key is test_row_0/C:col10/1731421067580/Put/seqid=0 2024-11-12T14:17:48,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742083_1259 (size=12711) 2024-11-12T14:17:49,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-12T14:17:49,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:49,000 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2902): Flushing 7a67a89740591358c1149d6b779d0225 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-12T14:17:49,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=A 2024-11-12T14:17:49,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:49,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=B 2024-11-12T14:17:49,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:49,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=C 2024-11-12T14:17:49,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:49,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/0763f48ba9704358926f31eae4344db5 is 50, key is test_row_0/A:col10/1731421068503/Put/seqid=0 2024-11-12T14:17:49,009 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742084_1260 (size=12301) 2024-11-12T14:17:49,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:49,151 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:49,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-12T14:17:49,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:49,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421129175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:49,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:49,180 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:49,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:49,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421129179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:49,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:49,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:49,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:49,292 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:49,292 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:49,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421129289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:49,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421129289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:49,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:49,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:49,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:49,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:49,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:49,292 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:49,370 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/b27a2b9141bc4b4a88beab9abfc62f7a as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/b27a2b9141bc4b4a88beab9abfc62f7a 2024-11-12T14:17:49,374 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 7a67a89740591358c1149d6b779d0225/C of 7a67a89740591358c1149d6b779d0225 into b27a2b9141bc4b4a88beab9abfc62f7a(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:49,375 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:49,375 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., storeName=7a67a89740591358c1149d6b779d0225/C, priority=12, startTime=1731421068911; duration=0sec 2024-11-12T14:17:49,375 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:49,375 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a67a89740591358c1149d6b779d0225:C 2024-11-12T14:17:49,410 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/0763f48ba9704358926f31eae4344db5 2024-11-12T14:17:49,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/922c6175ef4a4c37ada23c4eddf1556a is 50, key is test_row_0/B:col10/1731421068503/Put/seqid=0 2024-11-12T14:17:49,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742085_1261 (size=12301) 2024-11-12T14:17:49,421 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/922c6175ef4a4c37ada23c4eddf1556a 2024-11-12T14:17:49,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/1c0abdf4de8040e38c697331993ac13d is 50, key is test_row_0/C:col10/1731421068503/Put/seqid=0 2024-11-12T14:17:49,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742086_1262 (size=12301) 2024-11-12T14:17:49,449 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/1c0abdf4de8040e38c697331993ac13d 2024-11-12T14:17:49,453 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/0763f48ba9704358926f31eae4344db5 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/0763f48ba9704358926f31eae4344db5 2024-11-12T14:17:49,459 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/0763f48ba9704358926f31eae4344db5, entries=150, sequenceid=293, filesize=12.0 K 2024-11-12T14:17:49,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/922c6175ef4a4c37ada23c4eddf1556a as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/922c6175ef4a4c37ada23c4eddf1556a 2024-11-12T14:17:49,463 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/922c6175ef4a4c37ada23c4eddf1556a, entries=150, sequenceid=293, filesize=12.0 K 2024-11-12T14:17:49,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/1c0abdf4de8040e38c697331993ac13d as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/1c0abdf4de8040e38c697331993ac13d 2024-11-12T14:17:49,469 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/1c0abdf4de8040e38c697331993ac13d, entries=150, sequenceid=293, filesize=12.0 K 2024-11-12T14:17:49,470 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 7a67a89740591358c1149d6b779d0225 in 469ms, sequenceid=293, compaction requested=false 2024-11-12T14:17:49,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:49,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:49,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-12T14:17:49,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-12T14:17:49,473 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-12T14:17:49,473 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 928 msec 2024-11-12T14:17:49,475 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 934 msec 2024-11-12T14:17:49,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:49,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7a67a89740591358c1149d6b779d0225 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T14:17:49,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=A 2024-11-12T14:17:49,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:49,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=B 2024-11-12T14:17:49,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:49,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=C 2024-11-12T14:17:49,503 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:49,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/d62adb1f3bea4d0d94333424ae817720 is 50, key is test_row_0/A:col10/1731421069501/Put/seqid=0 2024-11-12T14:17:49,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742087_1263 (size=14741) 2024-11-12T14:17:49,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421129554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:49,558 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:49,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:49,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:49,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421129554, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:49,558 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:49,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:49,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:49,558 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:49,672 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:49,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421129669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:49,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:49,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:49,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:49,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:49,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421129669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:49,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:49,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:49,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-12T14:17:49,680 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:17:49,680 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:17:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-12T14:17:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-12T14:17:49,681 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:17:49,682 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:17:49,682 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:17:49,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:49,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421129739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:49,744 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_0' in 7a67a89740591358c1149d6b779d0225 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2160 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:17:49,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:49,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:49,744 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:49,762 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:49,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421129759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:49,763 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_2' in 7a67a89740591358c1149d6b779d0225 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:17:49,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:49,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:49,763 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:49,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-12T14:17:49,834 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-12T14:17:49,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:49,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:49,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:49,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:49,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:49,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:49,882 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:49,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421129879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:49,883 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:49,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:49,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421129879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:49,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:49,883 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:49,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:49,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:49,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:49,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/d62adb1f3bea4d0d94333424ae817720 2024-11-12T14:17:49,919 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/d1e23657b3724a1cacb0a0d20c99c480 is 50, key is test_row_0/B:col10/1731421069501/Put/seqid=0 2024-11-12T14:17:49,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742088_1264 (size=12301) 2024-11-12T14:17:49,987 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-12T14:17:49,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:49,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:49,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:49,987 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:49,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:49,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:50,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-12T14:17:50,142 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-12T14:17:50,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:50,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:50,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:50,143 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:50,143 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:50,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:50,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:50,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421130189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:50,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:50,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421130190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:50,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:50,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:50,194 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:50,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:50,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:50,195 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:50,296 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-12T14:17:50,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:50,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:50,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:50,296 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:50,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:50,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:50,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-12T14:17:50,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/d1e23657b3724a1cacb0a0d20c99c480 2024-11-12T14:17:50,329 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/6122378601504c898c12f5f21fc45580 is 50, key is test_row_0/C:col10/1731421069501/Put/seqid=0 2024-11-12T14:17:50,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742089_1265 (size=12301) 2024-11-12T14:17:50,449 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-12T14:17:50,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:50,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:50,449 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:50,449 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:50,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:50,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:50,602 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-12T14:17:50,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:50,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:50,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:50,602 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:50,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:50,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:50,701 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:50,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421130699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:50,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:50,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421130700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:50,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:50,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:50,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:50,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:50,702 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:50,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:50,734 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=312 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/6122378601504c898c12f5f21fc45580 2024-11-12T14:17:50,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/d62adb1f3bea4d0d94333424ae817720 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/d62adb1f3bea4d0d94333424ae817720 2024-11-12T14:17:50,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/d62adb1f3bea4d0d94333424ae817720, entries=200, sequenceid=312, filesize=14.4 K 2024-11-12T14:17:50,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/d1e23657b3724a1cacb0a0d20c99c480 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/d1e23657b3724a1cacb0a0d20c99c480 2024-11-12T14:17:50,755 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-12T14:17:50,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:50,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:50,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:50,755 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:50,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:50,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:50,761 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/d1e23657b3724a1cacb0a0d20c99c480, entries=150, sequenceid=312, filesize=12.0 K 2024-11-12T14:17:50,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/6122378601504c898c12f5f21fc45580 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/6122378601504c898c12f5f21fc45580 2024-11-12T14:17:50,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/6122378601504c898c12f5f21fc45580, entries=150, sequenceid=312, filesize=12.0 K 2024-11-12T14:17:50,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 7a67a89740591358c1149d6b779d0225 in 1267ms, sequenceid=312, compaction requested=true 2024-11-12T14:17:50,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:50,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a67a89740591358c1149d6b779d0225:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:17:50,769 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:50,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:50,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a67a89740591358c1149d6b779d0225:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:17:50,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:50,769 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:50,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a67a89740591358c1149d6b779d0225:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:17:50,769 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:50,770 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39753 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:50,770 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:50,770 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 7a67a89740591358c1149d6b779d0225/B is initiating minor compaction (all files) 2024-11-12T14:17:50,770 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 7a67a89740591358c1149d6b779d0225/A is initiating minor compaction (all files) 2024-11-12T14:17:50,770 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a67a89740591358c1149d6b779d0225/B in TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:50,770 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a67a89740591358c1149d6b779d0225/A in TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:50,770 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/59c61fc918354c03810426fae327f65e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/922c6175ef4a4c37ada23c4eddf1556a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/d1e23657b3724a1cacb0a0d20c99c480] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp, totalSize=36.4 K 2024-11-12T14:17:50,770 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/b210b2aeb99a4df59e26d5b13368e972, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/0763f48ba9704358926f31eae4344db5, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/d62adb1f3bea4d0d94333424ae817720] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp, totalSize=38.8 K 2024-11-12T14:17:50,771 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 59c61fc918354c03810426fae327f65e, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731421067551 2024-11-12T14:17:50,771 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b210b2aeb99a4df59e26d5b13368e972, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731421067551 2024-11-12T14:17:50,771 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 922c6175ef4a4c37ada23c4eddf1556a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731421068503 2024-11-12T14:17:50,772 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0763f48ba9704358926f31eae4344db5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731421068503 2024-11-12T14:17:50,772 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting d1e23657b3724a1cacb0a0d20c99c480, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731421069177 2024-11-12T14:17:50,772 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting d62adb1f3bea4d0d94333424ae817720, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731421069174 2024-11-12T14:17:50,781 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a67a89740591358c1149d6b779d0225#B#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:50,781 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/2ebe7d350b82415b8a54d78571b0aa27 is 50, key is test_row_0/B:col10/1731421069501/Put/seqid=0 2024-11-12T14:17:50,781 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a67a89740591358c1149d6b779d0225#A#compaction#220 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:50,781 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/fc7c1e3d517649c59769a4d350cf793c is 50, key is test_row_0/A:col10/1731421069501/Put/seqid=0 2024-11-12T14:17:50,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742090_1266 (size=12813) 2024-11-12T14:17:50,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742091_1267 (size=12813) 2024-11-12T14:17:50,817 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/fc7c1e3d517649c59769a4d350cf793c as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/fc7c1e3d517649c59769a4d350cf793c 2024-11-12T14:17:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-12T14:17:50,825 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a67a89740591358c1149d6b779d0225/A of 7a67a89740591358c1149d6b779d0225 into fc7c1e3d517649c59769a4d350cf793c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:50,825 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:50,825 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., storeName=7a67a89740591358c1149d6b779d0225/A, priority=13, startTime=1731421070769; duration=0sec 2024-11-12T14:17:50,825 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:50,825 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a67a89740591358c1149d6b779d0225:A 2024-11-12T14:17:50,826 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:50,827 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37313 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:50,828 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 7a67a89740591358c1149d6b779d0225/C is initiating minor compaction (all files) 2024-11-12T14:17:50,828 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a67a89740591358c1149d6b779d0225/C in TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:50,832 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/b27a2b9141bc4b4a88beab9abfc62f7a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/1c0abdf4de8040e38c697331993ac13d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/6122378601504c898c12f5f21fc45580] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp, totalSize=36.4 K 2024-11-12T14:17:50,833 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b27a2b9141bc4b4a88beab9abfc62f7a, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1731421067551 2024-11-12T14:17:50,833 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c0abdf4de8040e38c697331993ac13d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731421068503 2024-11-12T14:17:50,834 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6122378601504c898c12f5f21fc45580, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731421069177 2024-11-12T14:17:50,845 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a67a89740591358c1149d6b779d0225#C#compaction#221 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:50,845 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/e09188d2f7904f78a754d2d4fe05ea89 is 50, key is test_row_0/C:col10/1731421069501/Put/seqid=0 2024-11-12T14:17:50,861 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742092_1268 (size=12813) 2024-11-12T14:17:50,867 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/e09188d2f7904f78a754d2d4fe05ea89 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/e09188d2f7904f78a754d2d4fe05ea89 2024-11-12T14:17:50,872 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a67a89740591358c1149d6b779d0225/C of 7a67a89740591358c1149d6b779d0225 into e09188d2f7904f78a754d2d4fe05ea89(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:50,872 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:50,872 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., storeName=7a67a89740591358c1149d6b779d0225/C, priority=13, startTime=1731421070769; duration=0sec 2024-11-12T14:17:50,872 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:50,872 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a67a89740591358c1149d6b779d0225:C 2024-11-12T14:17:50,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-12T14:17:50,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:50,908 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 7a67a89740591358c1149d6b779d0225 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T14:17:50,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=A 2024-11-12T14:17:50,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:50,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=B 2024-11-12T14:17:50,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:50,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=C 2024-11-12T14:17:50,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:50,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/aa164e26a4734bdf9a70bf01402c421c is 50, key is test_row_0/A:col10/1731421069515/Put/seqid=0 2024-11-12T14:17:50,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742093_1269 (size=12301) 2024-11-12T14:17:50,918 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/aa164e26a4734bdf9a70bf01402c421c 2024-11-12T14:17:50,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/2728c696b57040e799c73f8d3e869555 is 50, key is test_row_0/B:col10/1731421069515/Put/seqid=0 2024-11-12T14:17:50,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742094_1270 (size=12301) 2024-11-12T14:17:51,210 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/2ebe7d350b82415b8a54d78571b0aa27 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/2ebe7d350b82415b8a54d78571b0aa27 2024-11-12T14:17:51,213 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a67a89740591358c1149d6b779d0225/B of 7a67a89740591358c1149d6b779d0225 into 2ebe7d350b82415b8a54d78571b0aa27(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:51,214 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:51,214 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., storeName=7a67a89740591358c1149d6b779d0225/B, priority=13, startTime=1731421070769; duration=0sec 2024-11-12T14:17:51,214 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:51,214 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a67a89740591358c1149d6b779d0225:B 2024-11-12T14:17:51,356 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/2728c696b57040e799c73f8d3e869555 2024-11-12T14:17:51,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/e25bdf12415c40ae87e510bbe4f0279b is 50, key is test_row_0/C:col10/1731421069515/Put/seqid=0 2024-11-12T14:17:51,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742095_1271 (size=12301) 2024-11-12T14:17:51,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:51,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:51,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:51,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421131747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421131746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:51,752 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:51,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:51,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421131759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:51,762 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_0' in 7a67a89740591358c1149d6b779d0225 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:17:51,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:51,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:51,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:51,766 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/e25bdf12415c40ae87e510bbe4f0279b 2024-11-12T14:17:51,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/aa164e26a4734bdf9a70bf01402c421c as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/aa164e26a4734bdf9a70bf01402c421c 2024-11-12T14:17:51,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:51,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421131769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:51,773 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_2' in 7a67a89740591358c1149d6b779d0225 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:17:51,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:51,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:51,773 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:51,774 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/aa164e26a4734bdf9a70bf01402c421c, entries=150, sequenceid=328, filesize=12.0 K 2024-11-12T14:17:51,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/2728c696b57040e799c73f8d3e869555 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/2728c696b57040e799c73f8d3e869555 2024-11-12T14:17:51,779 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/2728c696b57040e799c73f8d3e869555, entries=150, sequenceid=328, filesize=12.0 K 2024-11-12T14:17:51,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/e25bdf12415c40ae87e510bbe4f0279b as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/e25bdf12415c40ae87e510bbe4f0279b 2024-11-12T14:17:51,783 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/e25bdf12415c40ae87e510bbe4f0279b, entries=150, sequenceid=328, filesize=12.0 K 2024-11-12T14:17:51,784 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 7a67a89740591358c1149d6b779d0225 in 876ms, sequenceid=328, compaction requested=false 2024-11-12T14:17:51,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:51,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:51,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-12T14:17:51,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-12T14:17:51,787 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-12T14:17:51,787 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1040 sec 2024-11-12T14:17:51,788 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 2.1070 sec 2024-11-12T14:17:51,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-12T14:17:51,830 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:17:51,830 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:17:51,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees 2024-11-12T14:17:51,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T14:17:51,831 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:17:51,832 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=62, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:17:51,832 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:17:51,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:51,851 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7a67a89740591358c1149d6b779d0225 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T14:17:51,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=A 2024-11-12T14:17:51,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:51,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=B 2024-11-12T14:17:51,851 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:51,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=C 2024-11-12T14:17:51,852 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:51,855 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/31be70e86ec24d7ca0a6175a61fca74f is 50, key is test_row_0/A:col10/1731421071850/Put/seqid=0 2024-11-12T14:17:51,860 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742096_1272 (size=14741) 2024-11-12T14:17:51,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:51,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421131873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:51,878 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:51,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421131873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:51,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:51,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:51,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:51,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:51,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:51,879 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:51,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:51,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421131874, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:51,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:51,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:51,879 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:51,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T14:17:51,984 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-12T14:17:51,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:51,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:51,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:51,985 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:51,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:51,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:51,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:51,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421131989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:51,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:51,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:51,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421131989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:51,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:51,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:51,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:51,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:51,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:51,993 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:51,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421131989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:51,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:51,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:51,994 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:52,137 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-12T14:17:52,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:52,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:52,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:52,138 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:52,138 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:52,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:52,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T14:17:52,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:52,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421132199, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:52,201 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:52,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:52,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421132200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:52,202 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:52,202 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421132200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:52,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:52,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:52,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:52,202 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:52,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:52,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:52,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/31be70e86ec24d7ca0a6175a61fca74f 2024-11-12T14:17:52,266 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/7883cdccd2a84a9aa50fcc512e2d9cd6 is 50, key is test_row_0/B:col10/1731421071850/Put/seqid=0 2024-11-12T14:17:52,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742097_1273 (size=12301) 2024-11-12T14:17:52,290 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-12T14:17:52,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:52,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:52,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:52,290 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:52,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:52,443 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-12T14:17:52,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:52,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:52,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:52,443 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:52,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:52,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T14:17:52,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421132509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:52,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421132509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:52,512 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:52,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421132509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:52,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:52,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:52,513 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:52,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-12T14:17:52,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:52,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:52,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:52,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] handler.RSProcedureHandler(58): pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:52,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=63 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:52,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=63 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:52,672 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/7883cdccd2a84a9aa50fcc512e2d9cd6 2024-11-12T14:17:52,679 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/05fba06c0208424c99b22cd49d8642e6 is 50, key is test_row_0/C:col10/1731421071850/Put/seqid=0 2024-11-12T14:17:52,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742098_1274 (size=12301) 2024-11-12T14:17:52,683 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/05fba06c0208424c99b22cd49d8642e6 2024-11-12T14:17:52,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/31be70e86ec24d7ca0a6175a61fca74f as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/31be70e86ec24d7ca0a6175a61fca74f 2024-11-12T14:17:52,690 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/31be70e86ec24d7ca0a6175a61fca74f, entries=200, sequenceid=348, filesize=14.4 K 2024-11-12T14:17:52,691 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/7883cdccd2a84a9aa50fcc512e2d9cd6 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/7883cdccd2a84a9aa50fcc512e2d9cd6 2024-11-12T14:17:52,695 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/7883cdccd2a84a9aa50fcc512e2d9cd6, entries=150, sequenceid=348, filesize=12.0 K 2024-11-12T14:17:52,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/05fba06c0208424c99b22cd49d8642e6 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/05fba06c0208424c99b22cd49d8642e6 2024-11-12T14:17:52,698 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/05fba06c0208424c99b22cd49d8642e6, entries=150, sequenceid=348, filesize=12.0 K 2024-11-12T14:17:52,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 7a67a89740591358c1149d6b779d0225 in 848ms, sequenceid=348, compaction requested=true 2024-11-12T14:17:52,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:52,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a67a89740591358c1149d6b779d0225:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:17:52,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:52,700 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:52,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a67a89740591358c1149d6b779d0225:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:17:52,700 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:52,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:52,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a67a89740591358c1149d6b779d0225:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:17:52,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:52,700 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39855 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:52,700 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:52,700 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 7a67a89740591358c1149d6b779d0225/B is initiating minor compaction (all files) 2024-11-12T14:17:52,700 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 7a67a89740591358c1149d6b779d0225/A is initiating minor compaction (all files) 2024-11-12T14:17:52,701 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a67a89740591358c1149d6b779d0225/A in TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:52,701 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a67a89740591358c1149d6b779d0225/B in TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:52,701 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/2ebe7d350b82415b8a54d78571b0aa27, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/2728c696b57040e799c73f8d3e869555, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/7883cdccd2a84a9aa50fcc512e2d9cd6] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp, totalSize=36.5 K 2024-11-12T14:17:52,701 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/fc7c1e3d517649c59769a4d350cf793c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/aa164e26a4734bdf9a70bf01402c421c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/31be70e86ec24d7ca0a6175a61fca74f] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp, totalSize=38.9 K 2024-11-12T14:17:52,701 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting fc7c1e3d517649c59769a4d350cf793c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731421069177 2024-11-12T14:17:52,701 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ebe7d350b82415b8a54d78571b0aa27, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731421069177 2024-11-12T14:17:52,701 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2728c696b57040e799c73f8d3e869555, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731421069511 2024-11-12T14:17:52,701 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting aa164e26a4734bdf9a70bf01402c421c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731421069511 2024-11-12T14:17:52,701 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 31be70e86ec24d7ca0a6175a61fca74f, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731421071733 2024-11-12T14:17:52,701 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7883cdccd2a84a9aa50fcc512e2d9cd6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731421071733 2024-11-12T14:17:52,707 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a67a89740591358c1149d6b779d0225#B#compaction#228 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:52,707 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/36554949672048de8a61da96a395f741 is 50, key is test_row_0/B:col10/1731421071850/Put/seqid=0 2024-11-12T14:17:52,708 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a67a89740591358c1149d6b779d0225#A#compaction#229 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:52,708 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/cdd99a037e7b4024a0414603b860dda1 is 50, key is test_row_0/A:col10/1731421071850/Put/seqid=0 2024-11-12T14:17:52,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742099_1275 (size=12915) 2024-11-12T14:17:52,711 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742100_1276 (size=12915) 2024-11-12T14:17:52,748 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=63 2024-11-12T14:17:52,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:52,748 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2902): Flushing 7a67a89740591358c1149d6b779d0225 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T14:17:52,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=A 2024-11-12T14:17:52,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:52,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=B 2024-11-12T14:17:52,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:52,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=C 2024-11-12T14:17:52,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:52,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/7fae5c5124244f67a474b773f93e4753 is 50, key is test_row_0/A:col10/1731421071871/Put/seqid=0 2024-11-12T14:17:52,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742101_1277 (size=12301) 2024-11-12T14:17:52,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T14:17:53,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:53,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:53,057 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:53,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421133051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:53,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:53,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:53,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:53,058 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:53,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421133052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:53,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:53,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:53,058 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:53,061 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:53,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421133056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:53,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:53,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:53,062 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:53,115 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/cdd99a037e7b4024a0414603b860dda1 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/cdd99a037e7b4024a0414603b860dda1 2024-11-12T14:17:53,115 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/36554949672048de8a61da96a395f741 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/36554949672048de8a61da96a395f741 2024-11-12T14:17:53,119 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a67a89740591358c1149d6b779d0225/B of 7a67a89740591358c1149d6b779d0225 into 36554949672048de8a61da96a395f741(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:53,119 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:53,119 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., storeName=7a67a89740591358c1149d6b779d0225/B, priority=13, startTime=1731421072700; duration=0sec 2024-11-12T14:17:53,119 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:53,119 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a67a89740591358c1149d6b779d0225:B 2024-11-12T14:17:53,119 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:53,120 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37415 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:53,120 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 7a67a89740591358c1149d6b779d0225/C is initiating minor compaction (all files) 2024-11-12T14:17:53,120 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a67a89740591358c1149d6b779d0225/C in TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:53,121 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/e09188d2f7904f78a754d2d4fe05ea89, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/e25bdf12415c40ae87e510bbe4f0279b, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/05fba06c0208424c99b22cd49d8642e6] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp, totalSize=36.5 K 2024-11-12T14:17:53,121 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting e09188d2f7904f78a754d2d4fe05ea89, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=312, earliestPutTs=1731421069177 2024-11-12T14:17:53,121 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting e25bdf12415c40ae87e510bbe4f0279b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731421069511 2024-11-12T14:17:53,121 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05fba06c0208424c99b22cd49d8642e6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731421071733 2024-11-12T14:17:53,124 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a67a89740591358c1149d6b779d0225/A of 7a67a89740591358c1149d6b779d0225 into cdd99a037e7b4024a0414603b860dda1(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:53,124 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:53,125 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., storeName=7a67a89740591358c1149d6b779d0225/A, priority=13, startTime=1731421072700; duration=0sec 2024-11-12T14:17:53,125 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:53,125 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a67a89740591358c1149d6b779d0225:A 2024-11-12T14:17:53,129 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a67a89740591358c1149d6b779d0225#C#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:53,129 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/7da0380900184affa2df37ac9a06f91f is 50, key is test_row_0/C:col10/1731421071850/Put/seqid=0 2024-11-12T14:17:53,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742102_1278 (size=12915) 2024-11-12T14:17:53,139 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/7da0380900184affa2df37ac9a06f91f as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/7da0380900184affa2df37ac9a06f91f 2024-11-12T14:17:53,143 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a67a89740591358c1149d6b779d0225/C of 7a67a89740591358c1149d6b779d0225 into 7da0380900184affa2df37ac9a06f91f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:53,144 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:53,144 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., storeName=7a67a89740591358c1149d6b779d0225/C, priority=13, startTime=1731421072700; duration=0sec 2024-11-12T14:17:53,144 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:53,144 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a67a89740591358c1149d6b779d0225:C 2024-11-12T14:17:53,158 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/7fae5c5124244f67a474b773f93e4753 2024-11-12T14:17:53,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/370d7fadb98947d0aea698b8e53499a4 is 50, key is test_row_0/B:col10/1731421071871/Put/seqid=0 2024-11-12T14:17:53,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742103_1279 (size=12301) 2024-11-12T14:17:53,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:53,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421133169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:53,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:53,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:53,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421133169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:53,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:53,170 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:53,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:53,171 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:53,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:53,171 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:53,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421133170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:53,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:53,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:53,171 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:53,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:53,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421133379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:53,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:53,381 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:53,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:53,381 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:53,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421133379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:53,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:53,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:53,382 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:53,382 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:53,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421133379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:53,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:53,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:53,382 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:53,570 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/370d7fadb98947d0aea698b8e53499a4 2024-11-12T14:17:53,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/dfb30d178b5941e093da16f70c13b22d is 50, key is test_row_0/C:col10/1731421071871/Put/seqid=0 2024-11-12T14:17:53,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742104_1280 (size=12301) 2024-11-12T14:17:53,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:53,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421133689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:53,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:53,692 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:53,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:53,693 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:53,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421133689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:53,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:53,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:53,693 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:53,693 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:53,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421133690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:53,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:53,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:53,693 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:53,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T14:17:53,989 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=361 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/dfb30d178b5941e093da16f70c13b22d 2024-11-12T14:17:53,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/7fae5c5124244f67a474b773f93e4753 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/7fae5c5124244f67a474b773f93e4753 2024-11-12T14:17:53,996 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/7fae5c5124244f67a474b773f93e4753, entries=150, sequenceid=361, filesize=12.0 K 2024-11-12T14:17:53,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/370d7fadb98947d0aea698b8e53499a4 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/370d7fadb98947d0aea698b8e53499a4 2024-11-12T14:17:54,001 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/370d7fadb98947d0aea698b8e53499a4, entries=150, sequenceid=361, filesize=12.0 K 2024-11-12T14:17:54,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/dfb30d178b5941e093da16f70c13b22d as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/dfb30d178b5941e093da16f70c13b22d 2024-11-12T14:17:54,006 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/dfb30d178b5941e093da16f70c13b22d, entries=150, sequenceid=361, filesize=12.0 K 2024-11-12T14:17:54,006 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 7a67a89740591358c1149d6b779d0225 in 1258ms, sequenceid=361, compaction requested=false 2024-11-12T14:17:54,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.HRegion(2603): Flush status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:54,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:54,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=63}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=63 2024-11-12T14:17:54,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=63 2024-11-12T14:17:54,009 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-12T14:17:54,009 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1760 sec 2024-11-12T14:17:54,010 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=62, table=TestAcidGuarantees in 2.1790 sec 2024-11-12T14:17:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:54,202 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7a67a89740591358c1149d6b779d0225 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T14:17:54,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=A 2024-11-12T14:17:54,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:17:54,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=B 2024-11-12T14:17:54,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:17:54,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=C 2024-11-12T14:17:54,204 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:17:54,211 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/a0df0d4c0cad4410bbe98b35154e7031 is 50, key is test_row_0/A:col10/1731421073051/Put/seqid=0 2024-11-12T14:17:54,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742105_1281 (size=14741) 2024-11-12T14:17:54,216 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/a0df0d4c0cad4410bbe98b35154e7031 2024-11-12T14:17:54,227 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/9addd7bd0d734c7a813f06f31bacb3ba is 50, key is test_row_0/B:col10/1731421073051/Put/seqid=0 2024-11-12T14:17:54,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:54,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421134230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:54,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:54,236 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:54,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:54,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421134231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:54,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:54,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:54,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:54,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421134231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:54,237 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:54,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:54,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:54,237 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:54,238 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742106_1282 (size=12301) 2024-11-12T14:17:54,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:54,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421134339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:54,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:54,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421134339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:54,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:54,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:54,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:54,341 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:54,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:54,342 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:54,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:54,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421134349, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:54,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:54,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:54,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:54,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:54,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421134549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:54,551 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:54,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:54,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:54,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421134549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:54,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:54,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:54,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:54,552 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:54,562 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:54,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421134559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:54,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:54,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:54,562 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:54,639 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/9addd7bd0d734c7a813f06f31bacb3ba 2024-11-12T14:17:54,647 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/ac2d6061ccb7441aab75e60908eb3139 is 50, key is test_row_0/C:col10/1731421073051/Put/seqid=0 2024-11-12T14:17:54,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742107_1283 (size=12301) 2024-11-12T14:17:54,651 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=385 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/ac2d6061ccb7441aab75e60908eb3139 2024-11-12T14:17:54,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/a0df0d4c0cad4410bbe98b35154e7031 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/a0df0d4c0cad4410bbe98b35154e7031 2024-11-12T14:17:54,658 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/a0df0d4c0cad4410bbe98b35154e7031, entries=200, sequenceid=385, filesize=14.4 K 2024-11-12T14:17:54,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/9addd7bd0d734c7a813f06f31bacb3ba as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/9addd7bd0d734c7a813f06f31bacb3ba 2024-11-12T14:17:54,664 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/9addd7bd0d734c7a813f06f31bacb3ba, entries=150, sequenceid=385, filesize=12.0 K 2024-11-12T14:17:54,665 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/ac2d6061ccb7441aab75e60908eb3139 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/ac2d6061ccb7441aab75e60908eb3139 2024-11-12T14:17:54,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/ac2d6061ccb7441aab75e60908eb3139, entries=150, sequenceid=385, filesize=12.0 K 2024-11-12T14:17:54,672 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=60.38 KB/61830 for 7a67a89740591358c1149d6b779d0225 in 470ms, sequenceid=385, compaction requested=true 2024-11-12T14:17:54,672 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:54,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a67a89740591358c1149d6b779d0225:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:17:54,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:54,672 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:54,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a67a89740591358c1149d6b779d0225:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:17:54,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:54,672 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:54,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a67a89740591358c1149d6b779d0225:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:17:54,672 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:54,673 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:54,673 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39957 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:54,673 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 7a67a89740591358c1149d6b779d0225/A is initiating minor compaction (all files) 2024-11-12T14:17:54,673 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 7a67a89740591358c1149d6b779d0225/B is initiating minor compaction (all files) 2024-11-12T14:17:54,673 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a67a89740591358c1149d6b779d0225/A in TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:54,673 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a67a89740591358c1149d6b779d0225/B in TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:54,674 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/36554949672048de8a61da96a395f741, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/370d7fadb98947d0aea698b8e53499a4, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/9addd7bd0d734c7a813f06f31bacb3ba] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp, totalSize=36.6 K 2024-11-12T14:17:54,674 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/cdd99a037e7b4024a0414603b860dda1, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/7fae5c5124244f67a474b773f93e4753, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/a0df0d4c0cad4410bbe98b35154e7031] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp, totalSize=39.0 K 2024-11-12T14:17:54,674 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 36554949672048de8a61da96a395f741, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731421071733 2024-11-12T14:17:54,674 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdd99a037e7b4024a0414603b860dda1, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731421071733 2024-11-12T14:17:54,675 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 370d7fadb98947d0aea698b8e53499a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731421071867 2024-11-12T14:17:54,675 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7fae5c5124244f67a474b773f93e4753, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731421071867 2024-11-12T14:17:54,675 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 9addd7bd0d734c7a813f06f31bacb3ba, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731421073051 2024-11-12T14:17:54,675 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0df0d4c0cad4410bbe98b35154e7031, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731421073051 2024-11-12T14:17:54,693 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a67a89740591358c1149d6b779d0225#B#compaction#238 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:54,693 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/fcfe1320c7ac4c00867b477c411868fb is 50, key is test_row_0/B:col10/1731421073051/Put/seqid=0 2024-11-12T14:17:54,693 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a67a89740591358c1149d6b779d0225#A#compaction#237 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:54,694 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/f2fe86a8718f4e11893eff0256c8810a is 50, key is test_row_0/A:col10/1731421073051/Put/seqid=0 2024-11-12T14:17:54,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742109_1285 (size=13017) 2024-11-12T14:17:54,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742108_1284 (size=13017) 2024-11-12T14:17:54,702 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/fcfe1320c7ac4c00867b477c411868fb as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/fcfe1320c7ac4c00867b477c411868fb 2024-11-12T14:17:54,708 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/f2fe86a8718f4e11893eff0256c8810a as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/f2fe86a8718f4e11893eff0256c8810a 2024-11-12T14:17:54,708 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a67a89740591358c1149d6b779d0225/B of 7a67a89740591358c1149d6b779d0225 into fcfe1320c7ac4c00867b477c411868fb(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:54,708 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:54,708 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., storeName=7a67a89740591358c1149d6b779d0225/B, priority=13, startTime=1731421074672; duration=0sec 2024-11-12T14:17:54,709 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:54,709 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a67a89740591358c1149d6b779d0225:B 2024-11-12T14:17:54,709 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:54,711 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:54,711 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 7a67a89740591358c1149d6b779d0225/C is initiating minor compaction (all files) 2024-11-12T14:17:54,711 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a67a89740591358c1149d6b779d0225/C in TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:54,711 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/7da0380900184affa2df37ac9a06f91f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/dfb30d178b5941e093da16f70c13b22d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/ac2d6061ccb7441aab75e60908eb3139] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp, totalSize=36.6 K 2024-11-12T14:17:54,711 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 7da0380900184affa2df37ac9a06f91f, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731421071733 2024-11-12T14:17:54,712 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting dfb30d178b5941e093da16f70c13b22d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=361, earliestPutTs=1731421071867 2024-11-12T14:17:54,714 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting ac2d6061ccb7441aab75e60908eb3139, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731421073051 2024-11-12T14:17:54,718 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a67a89740591358c1149d6b779d0225/A of 7a67a89740591358c1149d6b779d0225 into f2fe86a8718f4e11893eff0256c8810a(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:54,718 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:54,718 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., storeName=7a67a89740591358c1149d6b779d0225/A, priority=13, startTime=1731421074672; duration=0sec 2024-11-12T14:17:54,718 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:54,719 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a67a89740591358c1149d6b779d0225:A 2024-11-12T14:17:54,727 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a67a89740591358c1149d6b779d0225#C#compaction#239 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:54,727 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/ef9d865ddc4a4fe38b287594b99f1857 is 50, key is test_row_0/C:col10/1731421073051/Put/seqid=0 2024-11-12T14:17:54,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742110_1286 (size=13017) 2024-11-12T14:17:54,745 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/ef9d865ddc4a4fe38b287594b99f1857 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/ef9d865ddc4a4fe38b287594b99f1857 2024-11-12T14:17:54,751 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a67a89740591358c1149d6b779d0225/C of 7a67a89740591358c1149d6b779d0225 into ef9d865ddc4a4fe38b287594b99f1857(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:54,751 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:54,752 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., storeName=7a67a89740591358c1149d6b779d0225/C, priority=13, startTime=1731421074672; duration=0sec 2024-11-12T14:17:54,752 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:54,752 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a67a89740591358c1149d6b779d0225:C 2024-11-12T14:17:54,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:54,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7a67a89740591358c1149d6b779d0225 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T14:17:54,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=A 2024-11-12T14:17:54,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:54,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=B 2024-11-12T14:17:54,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:54,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=C 2024-11-12T14:17:54,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:54,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421134883, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:54,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:54,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:54,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:54,888 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421134884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:54,888 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:54,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421134886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:54,889 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:54,892 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/5875d71ca7424f00ba43aab9d9e91507 is 50, key is test_row_0/A:col10/1731421074222/Put/seqid=0 2024-11-12T14:17:54,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742111_1287 (size=17181) 2024-11-12T14:17:55,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:55,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421135003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:55,006 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:55,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:55,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421135003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:55,007 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:55,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:55,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:55,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421135003, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:55,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:55,007 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:55,007 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:55,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:55,008 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:55,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:55,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421135209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:55,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:55,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:55,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:55,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421135209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:55,211 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:55,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:55,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:55,212 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:55,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:55,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421135219, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:55,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:55,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:55,221 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:55,296 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/5875d71ca7424f00ba43aab9d9e91507 2024-11-12T14:17:55,303 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/9a79c700b9e24b94b51e448df48f5fc7 is 50, key is test_row_0/B:col10/1731421074222/Put/seqid=0 2024-11-12T14:17:55,306 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742112_1288 (size=12301) 2024-11-12T14:17:55,520 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:55,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421135519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:55,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:55,521 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:55,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:55,521 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:55,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421135520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:55,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:55,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:55,521 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:55,530 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:55,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421135529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:55,530 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:55,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:55,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:55,707 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/9a79c700b9e24b94b51e448df48f5fc7 2024-11-12T14:17:55,713 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/c02934513e6342eb88ad74b54f982321 is 50, key is test_row_0/C:col10/1731421074222/Put/seqid=0 2024-11-12T14:17:55,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742113_1289 (size=12301) 2024-11-12T14:17:55,719 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=402 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/c02934513e6342eb88ad74b54f982321 2024-11-12T14:17:55,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/5875d71ca7424f00ba43aab9d9e91507 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/5875d71ca7424f00ba43aab9d9e91507 2024-11-12T14:17:55,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/5875d71ca7424f00ba43aab9d9e91507, entries=250, sequenceid=402, filesize=16.8 K 2024-11-12T14:17:55,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/9a79c700b9e24b94b51e448df48f5fc7 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/9a79c700b9e24b94b51e448df48f5fc7 2024-11-12T14:17:55,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/9a79c700b9e24b94b51e448df48f5fc7, entries=150, sequenceid=402, filesize=12.0 K 2024-11-12T14:17:55,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/c02934513e6342eb88ad74b54f982321 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/c02934513e6342eb88ad74b54f982321 2024-11-12T14:17:55,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/c02934513e6342eb88ad74b54f982321, entries=150, sequenceid=402, filesize=12.0 K 2024-11-12T14:17:55,751 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 7a67a89740591358c1149d6b779d0225 in 890ms, sequenceid=402, compaction requested=false 2024-11-12T14:17:55,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:55,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:55,802 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7a67a89740591358c1149d6b779d0225 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T14:17:55,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=A 2024-11-12T14:17:55,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:55,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=B 2024-11-12T14:17:55,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:55,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=C 2024-11-12T14:17:55,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:55,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/8ff46b38c4824207864b5c56f2801d24 is 50, key is test_row_0/A:col10/1731421075800/Put/seqid=0 2024-11-12T14:17:55,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742114_1290 (size=12301) 2024-11-12T14:17:55,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:55,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421135833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:55,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:55,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:55,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:55,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421135833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:55,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:55,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:55,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:55,837 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:55,943 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:55,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421135939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:55,943 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:55,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:55,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:55,943 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:55,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421135939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:55,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:55,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:55,943 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:55,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-12T14:17:55,999 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:17:55,999 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:17:56,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees 2024-11-12T14:17:56,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T14:17:56,002 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:17:56,002 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=64, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:17:56,002 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:17:56,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421136029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:56,032 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:56,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421136029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:56,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:56,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:56,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:56,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:56,033 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:56,033 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:56,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421136040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:56,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T14:17:56,152 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:56,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421136149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:56,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:56,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:56,152 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:56,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:56,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421136149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:56,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:56,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:56,153 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:56,154 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T14:17:56,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:56,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:56,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:56,154 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:56,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:56,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:56,212 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/8ff46b38c4824207864b5c56f2801d24 2024-11-12T14:17:56,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/d56f90a96c464bc1b8f659f3a302d390 is 50, key is test_row_0/B:col10/1731421075800/Put/seqid=0 2024-11-12T14:17:56,222 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742115_1291 (size=12301) 2024-11-12T14:17:56,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/d56f90a96c464bc1b8f659f3a302d390 2024-11-12T14:17:56,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/bd622497677348afad72b7d1e0b87008 is 50, key is test_row_0/C:col10/1731421075800/Put/seqid=0 2024-11-12T14:17:56,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742116_1292 (size=12301) 2024-11-12T14:17:56,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T14:17:56,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:56,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:56,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:56,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:56,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:56,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:56,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T14:17:56,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T14:17:56,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:56,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:56,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:56,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:56,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:56,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:56,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:56,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421136459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:56,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:56,463 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:56,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:56,463 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:56,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421136459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:56,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:56,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:56,463 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:56,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T14:17:56,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:56,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:56,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:56,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] handler.RSProcedureHandler(58): pid=65 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:56,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=65 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:56,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=65 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:56,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T14:17:56,635 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/bd622497677348afad72b7d1e0b87008 2024-11-12T14:17:56,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/8ff46b38c4824207864b5c56f2801d24 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/8ff46b38c4824207864b5c56f2801d24 2024-11-12T14:17:56,647 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/8ff46b38c4824207864b5c56f2801d24, entries=150, sequenceid=421, filesize=12.0 K 2024-11-12T14:17:56,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/d56f90a96c464bc1b8f659f3a302d390 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/d56f90a96c464bc1b8f659f3a302d390 2024-11-12T14:17:56,656 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/d56f90a96c464bc1b8f659f3a302d390, entries=150, sequenceid=421, filesize=12.0 K 2024-11-12T14:17:56,657 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/bd622497677348afad72b7d1e0b87008 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/bd622497677348afad72b7d1e0b87008 2024-11-12T14:17:56,662 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/bd622497677348afad72b7d1e0b87008, entries=150, sequenceid=421, filesize=12.0 K 2024-11-12T14:17:56,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=67.09 KB/68700 for 7a67a89740591358c1149d6b779d0225 in 860ms, sequenceid=421, compaction requested=true 2024-11-12T14:17:56,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:56,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a67a89740591358c1149d6b779d0225:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:17:56,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:56,663 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:56,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a67a89740591358c1149d6b779d0225:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:17:56,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:56,663 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:56,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a67a89740591358c1149d6b779d0225:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:17:56,663 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:56,663 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:56,663 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42499 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:56,664 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 7a67a89740591358c1149d6b779d0225/B is initiating minor compaction (all files) 2024-11-12T14:17:56,664 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 7a67a89740591358c1149d6b779d0225/A is initiating minor compaction (all files) 2024-11-12T14:17:56,664 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a67a89740591358c1149d6b779d0225/B in TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:56,664 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a67a89740591358c1149d6b779d0225/A in TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:56,664 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/fcfe1320c7ac4c00867b477c411868fb, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/9a79c700b9e24b94b51e448df48f5fc7, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/d56f90a96c464bc1b8f659f3a302d390] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp, totalSize=36.7 K 2024-11-12T14:17:56,664 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/f2fe86a8718f4e11893eff0256c8810a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/5875d71ca7424f00ba43aab9d9e91507, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/8ff46b38c4824207864b5c56f2801d24] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp, totalSize=41.5 K 2024-11-12T14:17:56,664 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting f2fe86a8718f4e11893eff0256c8810a, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731421073051 2024-11-12T14:17:56,664 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting fcfe1320c7ac4c00867b477c411868fb, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731421073051 2024-11-12T14:17:56,664 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5875d71ca7424f00ba43aab9d9e91507, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731421074208 2024-11-12T14:17:56,664 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a79c700b9e24b94b51e448df48f5fc7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731421074222 2024-11-12T14:17:56,665 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ff46b38c4824207864b5c56f2801d24, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731421074880 2024-11-12T14:17:56,665 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting d56f90a96c464bc1b8f659f3a302d390, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731421074880 2024-11-12T14:17:56,671 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a67a89740591358c1149d6b779d0225#B#compaction#246 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:56,672 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/18680df5e78a450fac478b388fa0efb4 is 50, key is test_row_0/B:col10/1731421075800/Put/seqid=0 2024-11-12T14:17:56,674 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a67a89740591358c1149d6b779d0225#A#compaction#247 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:56,674 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/a8fe381497924cd1b4b777db59ddcc37 is 50, key is test_row_0/A:col10/1731421075800/Put/seqid=0 2024-11-12T14:17:56,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742118_1294 (size=13119) 2024-11-12T14:17:56,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742117_1293 (size=13119) 2024-11-12T14:17:56,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=65 2024-11-12T14:17:56,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:56,765 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2902): Flushing 7a67a89740591358c1149d6b779d0225 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T14:17:56,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=A 2024-11-12T14:17:56,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:56,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=B 2024-11-12T14:17:56,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:56,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=C 2024-11-12T14:17:56,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:17:56,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/58b2a17c5b80404c98a7ccd0a50cca1d is 50, key is test_row_0/A:col10/1731421075831/Put/seqid=0 2024-11-12T14:17:56,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742119_1295 (size=12301) 2024-11-12T14:17:56,772 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/58b2a17c5b80404c98a7ccd0a50cca1d 2024-11-12T14:17:56,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/7a6791ff583d451a8de6743c050cf228 is 50, key is test_row_0/B:col10/1731421075831/Put/seqid=0 2024-11-12T14:17:56,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742120_1296 (size=12301) 2024-11-12T14:17:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:56,971 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:57,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:57,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421137010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:57,015 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:57,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:57,016 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:57,016 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:57,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421137012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:57,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:57,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421137039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:57,042 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_2' in 7a67a89740591358c1149d6b779d0225 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:17:57,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:57,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:57,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:57,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:57,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421137049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:57,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:57,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421137049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:57,052 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_0' in 7a67a89740591358c1149d6b779d0225 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:17:57,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:57,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:57,053 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_2' in 7a67a89740591358c1149d6b779d0225 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:17:57,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:57,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:57,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:57,053 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:57,087 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/a8fe381497924cd1b4b777db59ddcc37 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/a8fe381497924cd1b4b777db59ddcc37 2024-11-12T14:17:57,091 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a67a89740591358c1149d6b779d0225/A of 7a67a89740591358c1149d6b779d0225 into a8fe381497924cd1b4b777db59ddcc37(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:57,091 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:57,091 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., storeName=7a67a89740591358c1149d6b779d0225/A, priority=13, startTime=1731421076663; duration=0sec 2024-11-12T14:17:57,091 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:57,091 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a67a89740591358c1149d6b779d0225:A 2024-11-12T14:17:57,091 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:57,092 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:57,092 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 7a67a89740591358c1149d6b779d0225/C is initiating minor compaction (all files) 2024-11-12T14:17:57,092 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a67a89740591358c1149d6b779d0225/C in TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:57,092 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/ef9d865ddc4a4fe38b287594b99f1857, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/c02934513e6342eb88ad74b54f982321, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/bd622497677348afad72b7d1e0b87008] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp, totalSize=36.7 K 2024-11-12T14:17:57,093 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef9d865ddc4a4fe38b287594b99f1857, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=385, earliestPutTs=1731421073051 2024-11-12T14:17:57,093 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting c02934513e6342eb88ad74b54f982321, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=402, earliestPutTs=1731421074222 2024-11-12T14:17:57,093 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd622497677348afad72b7d1e0b87008, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731421074880 2024-11-12T14:17:57,098 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a67a89740591358c1149d6b779d0225#C#compaction#250 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:57,099 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/0778cfd69ca44d6dbf6ed5d5d0f4af2e is 50, key is test_row_0/C:col10/1731421075800/Put/seqid=0 2024-11-12T14:17:57,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742121_1297 (size=13119) 2024-11-12T14:17:57,104 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/18680df5e78a450fac478b388fa0efb4 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/18680df5e78a450fac478b388fa0efb4 2024-11-12T14:17:57,107 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a67a89740591358c1149d6b779d0225/B of 7a67a89740591358c1149d6b779d0225 into 18680df5e78a450fac478b388fa0efb4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:57,107 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:57,107 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., storeName=7a67a89740591358c1149d6b779d0225/B, priority=13, startTime=1731421076663; duration=0sec 2024-11-12T14:17:57,107 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:57,107 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a67a89740591358c1149d6b779d0225:B 2024-11-12T14:17:57,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421137119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:57,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:57,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:57,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421137120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:57,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:57,122 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:57,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:57,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:57,123 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:57,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T14:17:57,182 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/7a6791ff583d451a8de6743c050cf228 2024-11-12T14:17:57,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/efd772221f4f443aba1148fdeb6ade8f is 50, key is test_row_0/C:col10/1731421075831/Put/seqid=0 2024-11-12T14:17:57,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742122_1298 (size=12301) 2024-11-12T14:17:57,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421137329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:57,330 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:57,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421137329, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:57,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:57,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:57,330 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:57,330 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:57,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:57,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:57,507 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/0778cfd69ca44d6dbf6ed5d5d0f4af2e as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/0778cfd69ca44d6dbf6ed5d5d0f4af2e 2024-11-12T14:17:57,511 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a67a89740591358c1149d6b779d0225/C of 7a67a89740591358c1149d6b779d0225 into 0778cfd69ca44d6dbf6ed5d5d0f4af2e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:57,511 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:57,511 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., storeName=7a67a89740591358c1149d6b779d0225/C, priority=13, startTime=1731421076663; duration=0sec 2024-11-12T14:17:57,512 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:57,512 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a67a89740591358c1149d6b779d0225:C 2024-11-12T14:17:57,592 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=434 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/efd772221f4f443aba1148fdeb6ade8f 2024-11-12T14:17:57,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/58b2a17c5b80404c98a7ccd0a50cca1d as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/58b2a17c5b80404c98a7ccd0a50cca1d 2024-11-12T14:17:57,601 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/58b2a17c5b80404c98a7ccd0a50cca1d, entries=150, sequenceid=434, filesize=12.0 K 2024-11-12T14:17:57,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/7a6791ff583d451a8de6743c050cf228 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/7a6791ff583d451a8de6743c050cf228 2024-11-12T14:17:57,606 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/7a6791ff583d451a8de6743c050cf228, entries=150, sequenceid=434, filesize=12.0 K 2024-11-12T14:17:57,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/efd772221f4f443aba1148fdeb6ade8f as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/efd772221f4f443aba1148fdeb6ade8f 2024-11-12T14:17:57,612 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/efd772221f4f443aba1148fdeb6ade8f, entries=150, sequenceid=434, filesize=12.0 K 2024-11-12T14:17:57,613 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 7a67a89740591358c1149d6b779d0225 in 848ms, sequenceid=434, compaction requested=false 2024-11-12T14:17:57,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.HRegion(2603): Flush status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:57,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:57,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=65}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=65 2024-11-12T14:17:57,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=65 2024-11-12T14:17:57,618 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-12T14:17:57,619 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6150 sec 2024-11-12T14:17:57,620 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=64, table=TestAcidGuarantees in 1.6200 sec 2024-11-12T14:17:57,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:57,643 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7a67a89740591358c1149d6b779d0225 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T14:17:57,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731421076996, max timestamp=1731421077643Num uniques 859; 2024-11-12T14:17:57,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T14:17:57,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731421076996, max timestamp=1731421077643Num uniques 859; 2024-11-12T14:17:57,646 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T14:17:57,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731421076996, max timestamp=1731421077643Num uniques 859; 2024-11-12T14:17:57,648 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T14:17:57,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=A 2024-11-12T14:17:57,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:17:57,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=B 2024-11-12T14:17:57,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:17:57,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=C 2024-11-12T14:17:57,649 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:17:57,654 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/aeceab1165444a2a84dfeb2f56c4db1f is 50, key is test_row_0/A:col10/1731421077643/Put/seqid=0 2024-11-12T14:17:57,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742123_1299 (size=12301) 2024-11-12T14:17:57,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731421077682, max timestamp=1731421077713Num uniques 859; 2024-11-12T14:17:57,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731421077682, max timestamp=1731421077713Num uniques 859; 2024-11-12T14:17:57,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T14:17:57,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T14:17:57,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731421077682, max timestamp=1731421077713Num uniques 859; 2024-11-12T14:17:57,716 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T14:17:57,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7680, totalHeapSize=17712, min timestamp=1731421077745, max timestamp=1731421077769Num uniques 1068; 2024-11-12T14:17:57,773 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48900, new segment data size=7680, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-12T14:17:57,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731421077745, max timestamp=1731421077769Num uniques 1068; 2024-11-12T14:17:57,774 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-12T14:17:57,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=168, cellSize=7688, totalHeapSize=17712, min timestamp=1731421077745, max timestamp=1731421077769Num uniques 1068; 2024-11-12T14:17:57,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48908, new segment data size=7688, suffix heap size=139040,new segment heap size=17712 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1068, new segment cells count=168 2024-11-12T14:17:57,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731421077812, max timestamp=1731421077830Num uniques 1027; 2024-11-12T14:17:57,834 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-12T14:17:57,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731421077812, max timestamp=1731421077830Num uniques 1027; 2024-11-12T14:17:57,837 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-12T14:17:57,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=227, cellSize=10392, totalHeapSize=23848, min timestamp=1731421077812, max timestamp=1731421077830Num uniques 1027; 2024-11-12T14:17:57,841 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47032, new segment data size=10392, suffix heap size=134776,new segment heap size=23848 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1027, new segment cells count=227 2024-11-12T14:17:57,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731421077873, max timestamp=1731421077887Num uniques 1086; 2024-11-12T14:17:57,892 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-12T14:17:57,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=236, cellSize=10806, totalHeapSize=24784, min timestamp=1731421077867, max timestamp=1731421077887Num uniques 1086; 2024-11-12T14:17:57,896 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=10806, suffix heap size=140912,new segment heap size=24784 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=236 2024-11-12T14:17:57,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=186, cellSize=8516, totalHeapSize=19584, min timestamp=1731421077873, max timestamp=1731421077887Num uniques 1086; 2024-11-12T14:17:57,899 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49736, new segment data size=8516, suffix heap size=140912,new segment heap size=19584 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1086, new segment cells count=186 2024-11-12T14:17:57,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731421077949, max timestamp=1731421077966Num uniques 1045; 2024-11-12T14:17:57,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731421077949, max timestamp=1731421077966Num uniques 1095; 2024-11-12T14:17:57,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=245, cellSize=11220, totalHeapSize=25720, min timestamp=1731421077949, max timestamp=1731421077966Num uniques 1045; 2024-11-12T14:17:57,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50150, new segment data size=11220, suffix heap size=141848,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1095, new segment cells count=245 2024-11-12T14:17:57,970 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-12T14:17:57,971 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=47860, new segment data size=11220, suffix heap size=136648,new segment heap size=25720 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1045, new segment cells count=245 2024-11-12T14:17:58,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731421078004, max timestamp=1731421078046Num uniques 1104; 2024-11-12T14:17:58,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=204, cellSize=9340, totalHeapSize=21456, min timestamp=1731421078004, max timestamp=1731421078046Num uniques 1104; 2024-11-12T14:17:58,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-12T14:17:58,049 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=9340, suffix heap size=142784,new segment heap size=21456 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=204 2024-11-12T14:17:58,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=154, cellSize=7050, totalHeapSize=16256, min timestamp=1731421078016, max timestamp=1731421078046Num uniques 1104; 2024-11-12T14:17:58,053 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=50560, new segment data size=7050, suffix heap size=142784,new segment heap size=16256 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1104, new segment cells count=154 2024-11-12T14:17:58,074 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/aeceab1165444a2a84dfeb2f56c4db1f 2024-11-12T14:17:58,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/25ae97c5bf314e738542b8f7984812ea is 50, key is test_row_0/B:col10/1731421077643/Put/seqid=0 2024-11-12T14:17:58,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742124_1300 (size=12301) 2024-11-12T14:17:58,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731421078085, max timestamp=1731421078102Num uniques 1063; 2024-11-12T14:17:58,105 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-12T14:17:58,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731421078085, max timestamp=1731421078102Num uniques 1013; 2024-11-12T14:17:58,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=46388, new segment data size=9748, suffix heap size=133320,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1013, new segment cells count=213 2024-11-12T14:17:58,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=213, cellSize=9748, totalHeapSize=22392, min timestamp=1731421078085, max timestamp=1731421078102Num uniques 1063; 2024-11-12T14:17:58,106 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48678, new segment data size=9748, suffix heap size=138520,new segment heap size=22392 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1063, new segment cells count=213 2024-11-12T14:17:58,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=64 2024-11-12T14:17:58,150 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:17:58,150 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:17:58,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-12T14:17:58,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T14:17:58,152 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:17:58,152 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:17:58,152 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:17:58,156 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=222, cellSize=10162, totalHeapSize=23328, min timestamp=1731421078139, max timestamp=1731421078151Num uniques 1072; 2024-11-12T14:17:58,156 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=10162, suffix heap size=139456,new segment heap size=23328 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=222 2024-11-12T14:17:58,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=172, cellSize=7872, totalHeapSize=18128, min timestamp=1731421078144, max timestamp=1731421078151Num uniques 1072; 2024-11-12T14:17:58,159 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=7872, suffix heap size=139456,new segment heap size=18128 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=172 2024-11-12T14:17:58,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731421078144, max timestamp=1731421078151Num uniques 1072; 2024-11-12T14:17:58,163 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=49092, new segment data size=6870, suffix heap size=139456,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1072, new segment cells count=150 2024-11-12T14:17:58,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731421078193, max timestamp=1731421078205Num uniques 1031; 2024-11-12T14:17:58,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=47216, new segment data size=8286, suffix heap size=135192,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1031, new segment cells count=181 2024-11-12T14:17:58,209 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731421078193, max timestamp=1731421078205Num uniques 1009; 2024-11-12T14:17:58,210 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=181, cellSize=8286, totalHeapSize=19064, min timestamp=1731421078193, max timestamp=1731421078205Num uniques 1081; 2024-11-12T14:17:58,210 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46214, new segment data size=8286, suffix heap size=132904,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1009, new segment cells count=181 2024-11-12T14:17:58,210 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49506, new segment data size=8286, suffix heap size=140392,new segment heap size=19064 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1081, new segment cells count=181 2024-11-12T14:17:58,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T14:17:58,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8700, totalHeapSize=20000, min timestamp=1731421078239, max timestamp=1731421078264Num uniques 1040; 2024-11-12T14:17:58,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47630, new segment data size=8700, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-12T14:17:58,268 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8690, totalHeapSize=20000, min timestamp=1731421078239, max timestamp=1731421078264Num uniques 1040; 2024-11-12T14:17:58,269 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=47620, new segment data size=8690, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-12T14:17:58,272 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=190, cellSize=8690, totalHeapSize=20000, min timestamp=1731421078239, max timestamp=1731421078264Num uniques 1040; 2024-11-12T14:17:58,273 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=47620, new segment data size=8690, suffix heap size=136128,new segment heap size=20000 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1040, new segment cells count=190 2024-11-12T14:17:58,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T14:17:58,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:58,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:58,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:58,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:58,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:58,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:58,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731421078308, max timestamp=1731421078325Num uniques 1049; 2024-11-12T14:17:58,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731421078308, max timestamp=1731421078325Num uniques 1049; 2024-11-12T14:17:58,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=199, cellSize=9114, totalHeapSize=20936, min timestamp=1731421078308, max timestamp=1731421078325Num uniques 1049; 2024-11-12T14:17:58,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-12T14:17:58,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-12T14:17:58,330 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48044, new segment data size=9114, suffix heap size=137064,new segment heap size=20936 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1049, new segment cells count=199 2024-11-12T14:17:58,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731421078404, max timestamp=1731421078419Num uniques 1058; 2024-11-12T14:17:58,423 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=9520, suffix heap size=138000,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=208 2024-11-12T14:17:58,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731421078404, max timestamp=1731421078419Num uniques 1058; 2024-11-12T14:17:58,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=208, cellSize=9520, totalHeapSize=21872, min timestamp=1731421078404, max timestamp=1731421078419Num uniques 1058; 2024-11-12T14:17:58,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=9520, suffix heap size=138000,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=208 2024-11-12T14:17:58,424 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=48450, new segment data size=9520, suffix heap size=138000,new segment heap size=21872 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1058, new segment cells count=208 2024-11-12T14:17:58,457 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T14:17:58,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:58,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:58,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:58,458 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:58,458 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:58,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:58,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T14:17:58,475 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731421078438, max timestamp=1731421078472Num uniques 1067; 2024-11-12T14:17:58,475 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-11-12T14:17:58,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731421078438, max timestamp=1731421078472Num uniques 1067; 2024-11-12T14:17:58,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-11-12T14:17:58,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=167, cellSize=7642, totalHeapSize=17608, min timestamp=1731421078438, max timestamp=1731421078472Num uniques 1067; 2024-11-12T14:17:58,476 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=48862, new segment data size=7642, suffix heap size=138936,new segment heap size=17608 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1067, new segment cells count=167 2024-11-12T14:17:58,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/25ae97c5bf314e738542b8f7984812ea 2024-11-12T14:17:58,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/84e10e08a4b64a30aa960744cb394ce7 is 50, key is test_row_0/C:col10/1731421077643/Put/seqid=0 2024-11-12T14:17:58,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742125_1301 (size=12301) 2024-11-12T14:17:58,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1731421078507, max timestamp=1731421078520Num uniques 1026; 2024-11-12T14:17:58,523 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=8056, suffix heap size=134672,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=176 2024-11-12T14:17:58,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1731421078507, max timestamp=1731421078520Num uniques 1026; 2024-11-12T14:17:58,524 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=8056, suffix heap size=134672,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=176 2024-11-12T14:17:58,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=176, cellSize=8056, totalHeapSize=18544, min timestamp=1731421078507, max timestamp=1731421078520Num uniques 1026; 2024-11-12T14:17:58,526 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=46986, new segment data size=8056, suffix heap size=134672,new segment heap size=18544 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1026, new segment cells count=176 2024-11-12T14:17:58,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1731421078554, max timestamp=1731421078566Num uniques 1035; 2024-11-12T14:17:58,569 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=10760, suffix heap size=135608,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=235 2024-11-12T14:17:58,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1731421078554, max timestamp=1731421078566Num uniques 1035; 2024-11-12T14:17:58,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=10760, suffix heap size=135608,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=235 2024-11-12T14:17:58,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=235, cellSize=10760, totalHeapSize=24680, min timestamp=1731421078554, max timestamp=1731421078566Num uniques 1035; 2024-11-12T14:17:58,570 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47400, new segment data size=10760, suffix heap size=135608,new segment heap size=24680 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1035, new segment cells count=235 2024-11-12T14:17:58,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T14:17:58,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:58,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:58,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:58,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:58,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:58,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:58,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1731421078604, max timestamp=1731421078618Num uniques 1094; 2024-11-12T14:17:58,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=11174, suffix heap size=141744,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=244 2024-11-12T14:17:58,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1731421078604, max timestamp=1731421078618Num uniques 1094; 2024-11-12T14:17:58,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=11174, suffix heap size=141744,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=244 2024-11-12T14:17:58,622 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=244, cellSize=11174, totalHeapSize=25616, min timestamp=1731421078604, max timestamp=1731421078618Num uniques 1094; 2024-11-12T14:17:58,623 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=50104, new segment data size=11174, suffix heap size=141744,new segment heap size=25616 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1094, new segment cells count=244 2024-11-12T14:17:58,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1731421078680, max timestamp=1731421078694Num uniques 1103; 2024-11-12T14:17:58,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1731421078680, max timestamp=1731421078694Num uniques 1103; 2024-11-12T14:17:58,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=203, cellSize=9295, totalHeapSize=21352, min timestamp=1731421078680, max timestamp=1731421078694Num uniques 1103; 2024-11-12T14:17:58,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=9295, suffix heap size=142680,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=203 2024-11-12T14:17:58,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=9295, suffix heap size=142680,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=203 2024-11-12T14:17:58,698 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=50515, new segment data size=9295, suffix heap size=142680,new segment heap size=21352 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1103, new segment cells count=203 2024-11-12T14:17:58,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1731421078719, max timestamp=1731421078742Num uniques 1062; 2024-11-12T14:17:58,745 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=7412, suffix heap size=138416,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=162 2024-11-12T14:17:58,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1731421078719, max timestamp=1731421078742Num uniques 1062; 2024-11-12T14:17:58,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=162, cellSize=7412, totalHeapSize=17088, min timestamp=1731421078719, max timestamp=1731421078742Num uniques 1062; 2024-11-12T14:17:58,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=7412, suffix heap size=138416,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=162 2024-11-12T14:17:58,746 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=48632, new segment data size=7412, suffix heap size=138416,new segment heap size=17088 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1062, new segment cells count=162 2024-11-12T14:17:58,762 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T14:17:58,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:58,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:58,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:58,762 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:58,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:58,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:58,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T14:17:58,787 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=221, cellSize=10116, totalHeapSize=23224, min timestamp=1731421078771, max timestamp=1731421078783Num uniques 1021; 2024-11-12T14:17:58,787 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=221, cellSize=10116, totalHeapSize=23224, min timestamp=1731421078771, max timestamp=1731421078783Num uniques 1021; 2024-11-12T14:17:58,787 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=46756, new segment data size=10116, suffix heap size=134152,new segment heap size=23224 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1021, new segment cells count=221 2024-11-12T14:17:58,787 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=46756, new segment data size=10116, suffix heap size=134152,new segment heap size=23224 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1021, new segment cells count=221 2024-11-12T14:17:58,787 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=221, cellSize=10116, totalHeapSize=23224, min timestamp=1731421078771, max timestamp=1731421078783Num uniques 1021; 2024-11-12T14:17:58,787 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=46756, new segment data size=10116, suffix heap size=134152,new segment heap size=23224 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1021, new segment cells count=221 2024-11-12T14:17:58,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=180, cellSize=8240, totalHeapSize=18960, min timestamp=1731421078819, max timestamp=1731421078832Num uniques 1080; 2024-11-12T14:17:58,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=49460, new segment data size=8240, suffix heap size=140288,new segment heap size=18960 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1080, new segment cells count=180 2024-11-12T14:17:58,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1731421078812, max timestamp=1731421078832Num uniques 1080; 2024-11-12T14:17:58,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=49460, new segment data size=10530, suffix heap size=140288,new segment heap size=24160 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1080, new segment cells count=230 2024-11-12T14:17:58,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=230, cellSize=10530, totalHeapSize=24160, min timestamp=1731421078812, max timestamp=1731421078832Num uniques 1080; 2024-11-12T14:17:58,836 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=49460, new segment data size=10530, suffix heap size=140288,new segment heap size=24160 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1080, new segment cells count=230 2024-11-12T14:17:58,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=189, cellSize=8654, totalHeapSize=19896, min timestamp=1731421078869, max timestamp=1731421078876Num uniques 1039; 2024-11-12T14:17:58,879 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=47584, new segment data size=8654, suffix heap size=136024,new segment heap size=19896 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1039, new segment cells count=189 2024-11-12T14:17:58,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=239, cellSize=10944, totalHeapSize=25096, min timestamp=1731421078865, max timestamp=1731421078876Num uniques 1089; 2024-11-12T14:17:58,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=239, cellSize=10944, totalHeapSize=25096, min timestamp=1731421078865, max timestamp=1731421078876Num uniques 1089; 2024-11-12T14:17:58,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=49874, new segment data size=10944, suffix heap size=141224,new segment heap size=25096 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1089, new segment cells count=239 2024-11-12T14:17:58,880 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=49874, new segment data size=10944, suffix heap size=141224,new segment heap size=25096 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1089, new segment cells count=239 2024-11-12T14:17:58,911 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/84e10e08a4b64a30aa960744cb394ce7 2024-11-12T14:17:58,914 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T14:17:58,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:58,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:58,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:58,915 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:58,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:58,915 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/aeceab1165444a2a84dfeb2f56c4db1f as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/aeceab1165444a2a84dfeb2f56c4db1f 2024-11-12T14:17:58,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:58,920 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/aeceab1165444a2a84dfeb2f56c4db1f, entries=150, sequenceid=458, filesize=12.0 K 2024-11-12T14:17:58,921 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T14:17:58,921 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/25ae97c5bf314e738542b8f7984812ea as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/25ae97c5bf314e738542b8f7984812ea 2024-11-12T14:17:58,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=198, cellSize=9068, totalHeapSize=20832, min timestamp=1731421078902, max timestamp=1731421078918Num uniques 1098; 2024-11-12T14:17:58,922 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=50288, new segment data size=9068, suffix heap size=142160,new segment heap size=20832 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1098, new segment cells count=198 2024-11-12T14:17:58,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=198, cellSize=9068, totalHeapSize=20832, min timestamp=1731421078902, max timestamp=1731421078918Num uniques 1048; 2024-11-12T14:17:58,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=47998, new segment data size=9068, suffix heap size=136960,new segment heap size=20832 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1048, new segment cells count=198 2024-11-12T14:17:58,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=198, cellSize=9068, totalHeapSize=20832, min timestamp=1731421078902, max timestamp=1731421078918Num uniques 1098; 2024-11-12T14:17:58,923 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=50288, new segment data size=9068, suffix heap size=142160,new segment heap size=20832 　suffix off heap size=0, new segment off heap size=0, suffix cells count=1098, new segment cells count=198 2024-11-12T14:17:58,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/25ae97c5bf314e738542b8f7984812ea, entries=150, sequenceid=458, filesize=12.0 K 2024-11-12T14:17:58,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/84e10e08a4b64a30aa960744cb394ce7 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/84e10e08a4b64a30aa960744cb394ce7 2024-11-12T14:17:58,928 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/84e10e08a4b64a30aa960744cb394ce7, entries=150, sequenceid=458, filesize=12.0 K 2024-11-12T14:17:58,929 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~66.19 KB/67776, currentSize=33.54 KB/34350 for 7a67a89740591358c1149d6b779d0225 in 1286ms, sequenceid=458, compaction requested=true 2024-11-12T14:17:58,929 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:58,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a67a89740591358c1149d6b779d0225:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:17:58,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:58,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a67a89740591358c1149d6b779d0225:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:17:58,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:58,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 7a67a89740591358c1149d6b779d0225:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:17:58,930 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:58,930 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T14:17:58,930 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:58,931 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:58,931 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 7a67a89740591358c1149d6b779d0225/A is initiating minor compaction (all files) 2024-11-12T14:17:58,931 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a67a89740591358c1149d6b779d0225/A in TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:58,931 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/a8fe381497924cd1b4b777db59ddcc37, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/58b2a17c5b80404c98a7ccd0a50cca1d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/aeceab1165444a2a84dfeb2f56c4db1f] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp, totalSize=36.8 K 2024-11-12T14:17:58,932 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8fe381497924cd1b4b777db59ddcc37, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731421074880 2024-11-12T14:17:58,932 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:58,932 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 7a67a89740591358c1149d6b779d0225/B is initiating minor compaction (all files) 2024-11-12T14:17:58,932 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a67a89740591358c1149d6b779d0225/B in TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:58,932 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/18680df5e78a450fac478b388fa0efb4, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/7a6791ff583d451a8de6743c050cf228, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/25ae97c5bf314e738542b8f7984812ea] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp, totalSize=36.8 K 2024-11-12T14:17:58,932 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58b2a17c5b80404c98a7ccd0a50cca1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731421075823 2024-11-12T14:17:58,932 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 18680df5e78a450fac478b388fa0efb4, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731421074880 2024-11-12T14:17:58,933 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting aeceab1165444a2a84dfeb2f56c4db1f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1731421076996 2024-11-12T14:17:58,933 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a6791ff583d451a8de6743c050cf228, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731421075823 2024-11-12T14:17:58,934 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 25ae97c5bf314e738542b8f7984812ea, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1731421076996 2024-11-12T14:17:58,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 7a67a89740591358c1149d6b779d0225 2024-11-12T14:17:58,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 7a67a89740591358c1149d6b779d0225 3/3 column families, dataSize=53.67 KB heapSize=142.27 KB 2024-11-12T14:17:58,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=A 2024-11-12T14:17:58,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:17:58,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=B 2024-11-12T14:17:58,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:17:58,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=C 2024-11-12T14:17:58,936 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:17:58,941 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a67a89740591358c1149d6b779d0225#A#compaction#255 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:58,942 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a67a89740591358c1149d6b779d0225#B#compaction#256 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:58,942 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/bf771136374a4778b76a865f2efb3c17 is 50, key is test_row_0/A:col10/1731421077643/Put/seqid=0 2024-11-12T14:17:58,942 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/5f590a46e7c24202a4cc8cf23056dcc7 is 50, key is test_row_0/B:col10/1731421077643/Put/seqid=0 2024-11-12T14:17:58,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/3429a44812e54c2d88519d70177f6aee is 50, key is test_row_0/A:col10/1731421078934/Put/seqid=0 2024-11-12T14:17:58,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742126_1302 (size=13221) 2024-11-12T14:17:58,957 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742128_1304 (size=13221) 2024-11-12T14:17:58,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742127_1303 (size=14741) 2024-11-12T14:17:59,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731421078985, max timestamp=1731421079000Num uniques 859; 2024-11-12T14:17:59,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731421078985, max timestamp=1731421079000Num uniques 859; 2024-11-12T14:17:59,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T14:17:59,003 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T14:17:59,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731421078985, max timestamp=1731421079000Num uniques 859; 2024-11-12T14:17:59,005 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T14:17:59,056 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:59,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 333 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421139051, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:59,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:59,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:59,057 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:59,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:59,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 324 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421139053, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:59,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:59,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:59,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:59,064 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:59,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49198 deadline: 1731421139059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:59,065 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_2' in 7a67a89740591358c1149d6b779d0225 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:17:59,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:59,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:59,065 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:59,068 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T14:17:59,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:59,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:59,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:59,068 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:59,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:59,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:59,071 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:59,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49176 deadline: 1731421139069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:59,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:59,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49190 deadline: 1731421139069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:59,072 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_0' in 7a67a89740591358c1149d6b779d0225 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4185 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:17:59,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:59,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:59,072 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:59,072 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_2' in 7a67a89740591358c1149d6b779d0225 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:17:59,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:59,072 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:59,073 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:59,163 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:59,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 334 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421139159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:59,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:59,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:59,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:59,173 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:59,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 325 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421139169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:59,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:59,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:59,173 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:59,220 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T14:17:59,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:59,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:59,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:59,221 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:59,221 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:59,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:59,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T14:17:59,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=844 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/3429a44812e54c2d88519d70177f6aee 2024-11-12T14:17:59,370 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/bf771136374a4778b76a865f2efb3c17 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/bf771136374a4778b76a865f2efb3c17 2024-11-12T14:17:59,370 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/5f590a46e7c24202a4cc8cf23056dcc7 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/5f590a46e7c24202a4cc8cf23056dcc7 2024-11-12T14:17:59,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:59,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 335 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421139370, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:59,378 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:59,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T14:17:59,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:59,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:59,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:59,382 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:59,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:59,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:59,387 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:59,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 326 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421139380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:59,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:59,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:59,388 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:59,393 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a67a89740591358c1149d6b779d0225/A of 7a67a89740591358c1149d6b779d0225 into bf771136374a4778b76a865f2efb3c17(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:59,393 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:59,393 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., storeName=7a67a89740591358c1149d6b779d0225/A, priority=13, startTime=1731421078929; duration=0sec 2024-11-12T14:17:59,393 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:17:59,393 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a67a89740591358c1149d6b779d0225:A 2024-11-12T14:17:59,393 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:17:59,394 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a67a89740591358c1149d6b779d0225/B of 7a67a89740591358c1149d6b779d0225 into 5f590a46e7c24202a4cc8cf23056dcc7(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:59,394 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:59,394 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., storeName=7a67a89740591358c1149d6b779d0225/B, priority=13, startTime=1731421078930; duration=0sec 2024-11-12T14:17:59,394 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:59,394 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a67a89740591358c1149d6b779d0225:B 2024-11-12T14:17:59,394 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:17:59,394 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 7a67a89740591358c1149d6b779d0225/C is initiating minor compaction (all files) 2024-11-12T14:17:59,394 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 7a67a89740591358c1149d6b779d0225/C in TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:59,395 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/0778cfd69ca44d6dbf6ed5d5d0f4af2e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/efd772221f4f443aba1148fdeb6ade8f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/84e10e08a4b64a30aa960744cb394ce7] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp, totalSize=36.8 K 2024-11-12T14:17:59,396 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/7ef62c7c0fb24f9486841d7b1e63db10 is 50, key is test_row_0/B:col10/1731421078934/Put/seqid=0 2024-11-12T14:17:59,397 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0778cfd69ca44d6dbf6ed5d5d0f4af2e, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=421, earliestPutTs=1731421074880 2024-11-12T14:17:59,397 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting efd772221f4f443aba1148fdeb6ade8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=434, earliestPutTs=1731421075823 2024-11-12T14:17:59,398 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 84e10e08a4b64a30aa960744cb394ce7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1731421076996 2024-11-12T14:17:59,403 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742129_1305 (size=12301) 2024-11-12T14:17:59,404 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 7a67a89740591358c1149d6b779d0225#C#compaction#259 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:17:59,405 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/628bc4a61d664691ac4cf9366706d672 is 50, key is test_row_0/C:col10/1731421077643/Put/seqid=0 2024-11-12T14:17:59,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=844 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/7ef62c7c0fb24f9486841d7b1e63db10 2024-11-12T14:17:59,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742130_1306 (size=13221) 2024-11-12T14:17:59,415 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/bc393662e0d24299b34c1f677923437b is 50, key is test_row_0/C:col10/1731421078934/Put/seqid=0 2024-11-12T14:17:59,418 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/628bc4a61d664691ac4cf9366706d672 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/628bc4a61d664691ac4cf9366706d672 2024-11-12T14:17:59,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742131_1307 (size=12301) 2024-11-12T14:17:59,423 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 7a67a89740591358c1149d6b779d0225/C of 7a67a89740591358c1149d6b779d0225 into 628bc4a61d664691ac4cf9366706d672(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:17:59,423 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:59,423 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., storeName=7a67a89740591358c1149d6b779d0225/C, priority=13, startTime=1731421078930; duration=0sec 2024-11-12T14:17:59,424 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:17:59,424 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 7a67a89740591358c1149d6b779d0225:C 2024-11-12T14:17:59,536 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T14:17:59,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:59,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:59,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:59,536 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:59,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:59,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:59,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T14:17:59,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:59,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:17:59,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:59,689 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:59,689 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:59,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:17:59,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:59,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 336 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49170 deadline: 1731421139689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:59,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:59,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:59,694 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:59,701 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:17:59,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 327 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:49132 deadline: 1731421139700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:17:59,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:59,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:17:59,702 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:17:59,822 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=844 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/bc393662e0d24299b34c1f677923437b 2024-11-12T14:17:59,827 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/3429a44812e54c2d88519d70177f6aee as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/3429a44812e54c2d88519d70177f6aee 2024-11-12T14:17:59,833 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/3429a44812e54c2d88519d70177f6aee, entries=200, sequenceid=844, filesize=14.4 K 2024-11-12T14:17:59,834 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/7ef62c7c0fb24f9486841d7b1e63db10 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/7ef62c7c0fb24f9486841d7b1e63db10 2024-11-12T14:17:59,840 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/7ef62c7c0fb24f9486841d7b1e63db10, entries=150, sequenceid=844, filesize=12.0 K 2024-11-12T14:17:59,841 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/bc393662e0d24299b34c1f677923437b as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/bc393662e0d24299b34c1f677923437b 2024-11-12T14:17:59,846 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/bc393662e0d24299b34c1f677923437b, entries=150, sequenceid=844, filesize=12.0 K 2024-11-12T14:17:59,849 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~162.14 KB/166032, currentSize=127.47 KB/130530 for 7a67a89740591358c1149d6b779d0225 in 915ms, sequenceid=844, compaction requested=false 2024-11-12T14:17:59,849 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:17:59,852 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-12T14:17:59,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:17:59,852 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 7a67a89740591358c1149d6b779d0225 3/3 column families, dataSize=127.47 KB heapSize=360.38 KB 2024-11-12T14:17:59,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=A 2024-11-12T14:17:59,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:17:59,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=B 2024-11-12T14:17:59,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:17:59,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=C 2024-11-12T14:17:59,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:17:59,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/82fdd6cb9bb64a509e4689bc597fe13c is 50, key is test_row_0/A:col10/1731421079050/Put/seqid=0 2024-11-12T14:17:59,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742132_1308 (size=12301) 2024-11-12T14:17:59,915 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=882 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/82fdd6cb9bb64a509e4689bc597fe13c 2024-11-12T14:17:59,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/c6625d1ff25b415eb8a1ed795c0fbb53 is 50, key is test_row_0/B:col10/1731421079050/Put/seqid=0 2024-11-12T14:17:59,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742133_1309 (size=12301) 2024-11-12T14:18:00,090 INFO [Thread-1053 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1053. 2024-11-12T14:18:00,090 DEBUG [Thread-1053 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:18:00,090 DEBUG [Thread-1053 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:00,090 DEBUG [Thread-1053 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:00,092 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:18:00,093 INFO [Thread-1059 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1059. 2024-11-12T14:18:00,093 DEBUG [Thread-1059 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:18:00,093 DEBUG [Thread-1059 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:00,093 INFO [Thread-1055 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1055. 2024-11-12T14:18:00,093 DEBUG [Thread-1055 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:18:00,093 DEBUG [Thread-1055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:00,093 DEBUG [Thread-1055 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:00,093 INFO [Thread-1057 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1057. 2024-11-12T14:18:00,093 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:18:00,093 DEBUG [Thread-1057 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:18:00,093 DEBUG [Thread-1057 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:00,093 DEBUG [Thread-1057 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:00,093 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:18:00,096 DEBUG [Thread-1059 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:00,096 INFO [Thread-1051 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1051. 2024-11-12T14:18:00,096 DEBUG [Thread-1051 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:18:00,096 DEBUG [Thread-1051 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:00,096 DEBUG [Thread-1051 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:00,100 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:18:00,101 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:18:00,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 7a67a89740591358c1149d6b779d0225 2024-11-12T14:18:00,200 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. as already flushing 2024-11-12T14:18:00,201 INFO [Thread-1044 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1044. 2024-11-12T14:18:00,201 DEBUG [Thread-1044 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:18:00,201 DEBUG [Thread-1044 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:00,201 DEBUG [Thread-1044 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:00,201 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:18:00,211 INFO [Thread-1042 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1042. 2024-11-12T14:18:00,211 DEBUG [Thread-1042 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:18:00,211 DEBUG [Thread-1042 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:00,211 DEBUG [Thread-1042 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:00,212 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:18:00,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T14:18:00,336 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=882 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/c6625d1ff25b415eb8a1ed795c0fbb53 2024-11-12T14:18:00,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/a46abf23f36145b481ae92e152f80cea is 50, key is test_row_0/C:col10/1731421079050/Put/seqid=0 2024-11-12T14:18:00,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742134_1310 (size=12301) 2024-11-12T14:18:00,758 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=882 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/a46abf23f36145b481ae92e152f80cea 2024-11-12T14:18:00,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/82fdd6cb9bb64a509e4689bc597fe13c as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/82fdd6cb9bb64a509e4689bc597fe13c 2024-11-12T14:18:00,768 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/82fdd6cb9bb64a509e4689bc597fe13c, entries=150, sequenceid=882, filesize=12.0 K 2024-11-12T14:18:00,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/c6625d1ff25b415eb8a1ed795c0fbb53 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/c6625d1ff25b415eb8a1ed795c0fbb53 2024-11-12T14:18:00,774 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/c6625d1ff25b415eb8a1ed795c0fbb53, entries=150, sequenceid=882, filesize=12.0 K 2024-11-12T14:18:00,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/a46abf23f36145b481ae92e152f80cea as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/a46abf23f36145b481ae92e152f80cea 2024-11-12T14:18:00,780 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/a46abf23f36145b481ae92e152f80cea, entries=150, sequenceid=882, filesize=12.0 K 2024-11-12T14:18:00,781 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~360.33 KB/368976, currentSize=13.42 KB/13740 for 7a67a89740591358c1149d6b779d0225 in 929ms, sequenceid=882, compaction requested=true 2024-11-12T14:18:00,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 7a67a89740591358c1149d6b779d0225: 2024-11-12T14:18:00,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:18:00,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-12T14:18:00,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-12T14:18:00,784 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-12T14:18:00,785 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6310 sec 2024-11-12T14:18:00,786 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 2.6350 sec 2024-11-12T14:18:02,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-12T14:18:02,310 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:18:03,091 INFO [Thread-1040 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1040. 2024-11-12T14:18:03,091 DEBUG [Thread-1040 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:18:03,091 DEBUG [Thread-1040 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:03,091 DEBUG [Thread-1040 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:03,091 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:18:03,111 INFO [Thread-1048 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1048. 2024-11-12T14:18:03,111 DEBUG [Thread-1048 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:18:03,111 DEBUG [Thread-1048 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:03,111 DEBUG [Thread-1048 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:03,111 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:18:03,120 INFO [Thread-1046 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1046. 2024-11-12T14:18:03,121 DEBUG [Thread-1046 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:18:03,121 DEBUG [Thread-1046 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:03,121 DEBUG [Thread-1046 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:03,121 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:18:03,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T14:18:03,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-12T14:18:03,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 292 2024-11-12T14:18:03,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 301 2024-11-12T14:18:03,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 59 2024-11-12T14:18:03,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 89 2024-11-12T14:18:03,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T14:18:03,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2501 2024-11-12T14:18:03,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2469 2024-11-12T14:18:03,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2442 2024-11-12T14:18:03,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2514 2024-11-12T14:18:03,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2461 2024-11-12T14:18:03,121 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T14:18:03,121 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T14:18:03,121 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T14:18:03,121 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:03,121 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:03,121 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:18:03,122 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-12T14:18:03,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T14:18:03,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T14:18:03,126 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731421083126"}]},"ts":"1731421083126"} 2024-11-12T14:18:03,128 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T14:18:03,128 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T14:18:03,129 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T14:18:03,130 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7a67a89740591358c1149d6b779d0225, UNASSIGN}] 2024-11-12T14:18:03,131 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7a67a89740591358c1149d6b779d0225, UNASSIGN 2024-11-12T14:18:03,132 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=7a67a89740591358c1149d6b779d0225, regionState=CLOSING, regionLocation=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:03,134 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=70, ppid=69, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=7a67a89740591358c1149d6b779d0225, UNASSIGN because future has completed 2024-11-12T14:18:03,134 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T14:18:03,134 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822}] 2024-11-12T14:18:03,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T14:18:03,287 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] handler.UnassignRegionHandler(122): Close 7a67a89740591358c1149d6b779d0225 2024-11-12T14:18:03,287 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T14:18:03,287 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegion(1722): Closing 7a67a89740591358c1149d6b779d0225, disabling compactions & flushes 2024-11-12T14:18:03,287 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:18:03,287 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:18:03,287 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. after waiting 0 ms 2024-11-12T14:18:03,287 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:18:03,287 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegion(2902): Flushing 7a67a89740591358c1149d6b779d0225 3/3 column families, dataSize=33.54 KB heapSize=100.36 KB 2024-11-12T14:18:03,288 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=A 2024-11-12T14:18:03,288 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:03,288 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=B 2024-11-12T14:18:03,288 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:03,288 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 7a67a89740591358c1149d6b779d0225, store=C 2024-11-12T14:18:03,288 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:03,297 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/a04d3dbf6dd747989d529b262a97f239 is 50, key is test_row_0/A:col10/1731421083119/Put/seqid=0 2024-11-12T14:18:03,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742135_1311 (size=12301) 2024-11-12T14:18:03,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T14:18:03,705 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=890 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/a04d3dbf6dd747989d529b262a97f239 2024-11-12T14:18:03,714 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/0bd27edda1f54be0b0a702c540704785 is 50, key is test_row_0/B:col10/1731421083119/Put/seqid=0 2024-11-12T14:18:03,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742136_1312 (size=12301) 2024-11-12T14:18:03,728 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=890 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/0bd27edda1f54be0b0a702c540704785 2024-11-12T14:18:03,740 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/6588322c3af341ffa2b3c3a4ec69fe07 is 50, key is test_row_0/C:col10/1731421083119/Put/seqid=0 2024-11-12T14:18:03,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T14:18:03,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742137_1313 (size=12301) 2024-11-12T14:18:04,157 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=890 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/6588322c3af341ffa2b3c3a4ec69fe07 2024-11-12T14:18:04,172 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/A/a04d3dbf6dd747989d529b262a97f239 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/a04d3dbf6dd747989d529b262a97f239 2024-11-12T14:18:04,179 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/a04d3dbf6dd747989d529b262a97f239, entries=150, sequenceid=890, filesize=12.0 K 2024-11-12T14:18:04,180 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/B/0bd27edda1f54be0b0a702c540704785 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/0bd27edda1f54be0b0a702c540704785 2024-11-12T14:18:04,191 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/0bd27edda1f54be0b0a702c540704785, entries=150, sequenceid=890, filesize=12.0 K 2024-11-12T14:18:04,192 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/.tmp/C/6588322c3af341ffa2b3c3a4ec69fe07 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/6588322c3af341ffa2b3c3a4ec69fe07 2024-11-12T14:18:04,199 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/6588322c3af341ffa2b3c3a4ec69fe07, entries=150, sequenceid=890, filesize=12.0 K 2024-11-12T14:18:04,207 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 7a67a89740591358c1149d6b779d0225 in 920ms, sequenceid=890, compaction requested=true 2024-11-12T14:18:04,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/35ae6d1dee954d0d9ac630030115b152, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/b00ff763a1cb499eb2548ffbe473a361, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/0ebded243c2d45d0a580f037a8e0b9c5, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/629954e0385c4cd4a045fc3141d31f8a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/db2e6faf2ea74eff88a59dc7b07a1d98, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/afa5a740bbbc4e4a9f81397e53e9fd55, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/a897c7ed81cc451c9a4929ecc527cbf1, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/1809e3f6feab46408316b04e0e2c65a7, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/3bd71df898a043f9bc0ae43c20e9bb86, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/c234cc08fe9c44eabd9ceca3de03dcf9, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/be3432373b02415388b52efcd8ee16d8, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/0052da1763114897a7dfad3db15c2bde, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/b210b2aeb99a4df59e26d5b13368e972, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/0763f48ba9704358926f31eae4344db5, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/d62adb1f3bea4d0d94333424ae817720, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/fc7c1e3d517649c59769a4d350cf793c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/aa164e26a4734bdf9a70bf01402c421c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/31be70e86ec24d7ca0a6175a61fca74f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/cdd99a037e7b4024a0414603b860dda1, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/7fae5c5124244f67a474b773f93e4753, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/a0df0d4c0cad4410bbe98b35154e7031, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/f2fe86a8718f4e11893eff0256c8810a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/5875d71ca7424f00ba43aab9d9e91507, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/a8fe381497924cd1b4b777db59ddcc37, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/8ff46b38c4824207864b5c56f2801d24, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/58b2a17c5b80404c98a7ccd0a50cca1d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/aeceab1165444a2a84dfeb2f56c4db1f] to archive 2024-11-12T14:18:04,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T14:18:04,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/35ae6d1dee954d0d9ac630030115b152 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/35ae6d1dee954d0d9ac630030115b152 2024-11-12T14:18:04,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/b00ff763a1cb499eb2548ffbe473a361 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/b00ff763a1cb499eb2548ffbe473a361 2024-11-12T14:18:04,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/0ebded243c2d45d0a580f037a8e0b9c5 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/0ebded243c2d45d0a580f037a8e0b9c5 2024-11-12T14:18:04,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/629954e0385c4cd4a045fc3141d31f8a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/629954e0385c4cd4a045fc3141d31f8a 2024-11-12T14:18:04,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/db2e6faf2ea74eff88a59dc7b07a1d98 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/db2e6faf2ea74eff88a59dc7b07a1d98 2024-11-12T14:18:04,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/afa5a740bbbc4e4a9f81397e53e9fd55 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/afa5a740bbbc4e4a9f81397e53e9fd55 2024-11-12T14:18:04,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/a897c7ed81cc451c9a4929ecc527cbf1 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/a897c7ed81cc451c9a4929ecc527cbf1 2024-11-12T14:18:04,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/1809e3f6feab46408316b04e0e2c65a7 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/1809e3f6feab46408316b04e0e2c65a7 2024-11-12T14:18:04,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/3bd71df898a043f9bc0ae43c20e9bb86 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/3bd71df898a043f9bc0ae43c20e9bb86 2024-11-12T14:18:04,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/c234cc08fe9c44eabd9ceca3de03dcf9 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/c234cc08fe9c44eabd9ceca3de03dcf9 2024-11-12T14:18:04,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/be3432373b02415388b52efcd8ee16d8 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/be3432373b02415388b52efcd8ee16d8 2024-11-12T14:18:04,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/0052da1763114897a7dfad3db15c2bde to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/0052da1763114897a7dfad3db15c2bde 2024-11-12T14:18:04,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/b210b2aeb99a4df59e26d5b13368e972 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/b210b2aeb99a4df59e26d5b13368e972 2024-11-12T14:18:04,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/0763f48ba9704358926f31eae4344db5 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/0763f48ba9704358926f31eae4344db5 2024-11-12T14:18:04,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/d62adb1f3bea4d0d94333424ae817720 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/d62adb1f3bea4d0d94333424ae817720 2024-11-12T14:18:04,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/fc7c1e3d517649c59769a4d350cf793c to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/fc7c1e3d517649c59769a4d350cf793c 2024-11-12T14:18:04,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/aa164e26a4734bdf9a70bf01402c421c to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/aa164e26a4734bdf9a70bf01402c421c 2024-11-12T14:18:04,259 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/31be70e86ec24d7ca0a6175a61fca74f to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/31be70e86ec24d7ca0a6175a61fca74f 2024-11-12T14:18:04,260 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/cdd99a037e7b4024a0414603b860dda1 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/cdd99a037e7b4024a0414603b860dda1 2024-11-12T14:18:04,261 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/7fae5c5124244f67a474b773f93e4753 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/7fae5c5124244f67a474b773f93e4753 2024-11-12T14:18:04,263 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/a0df0d4c0cad4410bbe98b35154e7031 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/a0df0d4c0cad4410bbe98b35154e7031 2024-11-12T14:18:04,264 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/f2fe86a8718f4e11893eff0256c8810a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/f2fe86a8718f4e11893eff0256c8810a 2024-11-12T14:18:04,265 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/5875d71ca7424f00ba43aab9d9e91507 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/5875d71ca7424f00ba43aab9d9e91507 2024-11-12T14:18:04,266 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/a8fe381497924cd1b4b777db59ddcc37 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/a8fe381497924cd1b4b777db59ddcc37 2024-11-12T14:18:04,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T14:18:04,276 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/8ff46b38c4824207864b5c56f2801d24 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/8ff46b38c4824207864b5c56f2801d24 2024-11-12T14:18:04,279 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/58b2a17c5b80404c98a7ccd0a50cca1d to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/58b2a17c5b80404c98a7ccd0a50cca1d 2024-11-12T14:18:04,281 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/aeceab1165444a2a84dfeb2f56c4db1f to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/aeceab1165444a2a84dfeb2f56c4db1f 2024-11-12T14:18:04,282 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/69946287a9eb4ddaada2e2959c0a7774, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/c15c09973b9246ccbd052d5cf9314784, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/c2ca0410932943109c04eb0d79db248e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/10c9f612a8e24d1aa5df715b501ad97c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/20883ef18bee422abfadcfdfff4ba709, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/6c78d05c25194b2b9ca6032eba87ca74, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/ee0758775ce04336b1097822c452540e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/0e7f564e0224424ab7eb1338a5c7e814, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/21bdfdfbef9b41a9bfc220897b7fc44c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/f1d73a37294c452bb6f9923149f03c90, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/05ab3bfba2e04a85be974a4106e7669c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/59c61fc918354c03810426fae327f65e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/3ee3aa87f7a24254ab586675730ac11d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/922c6175ef4a4c37ada23c4eddf1556a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/2ebe7d350b82415b8a54d78571b0aa27, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/d1e23657b3724a1cacb0a0d20c99c480, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/2728c696b57040e799c73f8d3e869555, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/36554949672048de8a61da96a395f741, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/7883cdccd2a84a9aa50fcc512e2d9cd6, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/370d7fadb98947d0aea698b8e53499a4, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/fcfe1320c7ac4c00867b477c411868fb, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/9addd7bd0d734c7a813f06f31bacb3ba, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/9a79c700b9e24b94b51e448df48f5fc7, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/18680df5e78a450fac478b388fa0efb4, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/d56f90a96c464bc1b8f659f3a302d390, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/7a6791ff583d451a8de6743c050cf228, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/25ae97c5bf314e738542b8f7984812ea] to archive 2024-11-12T14:18:04,283 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T14:18:04,285 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/69946287a9eb4ddaada2e2959c0a7774 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/69946287a9eb4ddaada2e2959c0a7774 2024-11-12T14:18:04,286 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/c15c09973b9246ccbd052d5cf9314784 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/c15c09973b9246ccbd052d5cf9314784 2024-11-12T14:18:04,288 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/c2ca0410932943109c04eb0d79db248e to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/c2ca0410932943109c04eb0d79db248e 2024-11-12T14:18:04,301 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/10c9f612a8e24d1aa5df715b501ad97c to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/10c9f612a8e24d1aa5df715b501ad97c 2024-11-12T14:18:04,304 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/20883ef18bee422abfadcfdfff4ba709 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/20883ef18bee422abfadcfdfff4ba709 2024-11-12T14:18:04,305 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/6c78d05c25194b2b9ca6032eba87ca74 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/6c78d05c25194b2b9ca6032eba87ca74 2024-11-12T14:18:04,306 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/ee0758775ce04336b1097822c452540e to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/ee0758775ce04336b1097822c452540e 2024-11-12T14:18:04,308 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/0e7f564e0224424ab7eb1338a5c7e814 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/0e7f564e0224424ab7eb1338a5c7e814 2024-11-12T14:18:04,309 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/21bdfdfbef9b41a9bfc220897b7fc44c to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/21bdfdfbef9b41a9bfc220897b7fc44c 2024-11-12T14:18:04,311 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/f1d73a37294c452bb6f9923149f03c90 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/f1d73a37294c452bb6f9923149f03c90 2024-11-12T14:18:04,312 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/05ab3bfba2e04a85be974a4106e7669c to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/05ab3bfba2e04a85be974a4106e7669c 2024-11-12T14:18:04,315 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/59c61fc918354c03810426fae327f65e to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/59c61fc918354c03810426fae327f65e 2024-11-12T14:18:04,316 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/3ee3aa87f7a24254ab586675730ac11d to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/3ee3aa87f7a24254ab586675730ac11d 2024-11-12T14:18:04,318 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/922c6175ef4a4c37ada23c4eddf1556a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/922c6175ef4a4c37ada23c4eddf1556a 2024-11-12T14:18:04,319 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/2ebe7d350b82415b8a54d78571b0aa27 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/2ebe7d350b82415b8a54d78571b0aa27 2024-11-12T14:18:04,320 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/d1e23657b3724a1cacb0a0d20c99c480 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/d1e23657b3724a1cacb0a0d20c99c480 2024-11-12T14:18:04,322 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/2728c696b57040e799c73f8d3e869555 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/2728c696b57040e799c73f8d3e869555 2024-11-12T14:18:04,323 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/36554949672048de8a61da96a395f741 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/36554949672048de8a61da96a395f741 2024-11-12T14:18:04,325 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/7883cdccd2a84a9aa50fcc512e2d9cd6 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/7883cdccd2a84a9aa50fcc512e2d9cd6 2024-11-12T14:18:04,326 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/370d7fadb98947d0aea698b8e53499a4 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/370d7fadb98947d0aea698b8e53499a4 2024-11-12T14:18:04,327 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/fcfe1320c7ac4c00867b477c411868fb to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/fcfe1320c7ac4c00867b477c411868fb 2024-11-12T14:18:04,329 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/9addd7bd0d734c7a813f06f31bacb3ba to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/9addd7bd0d734c7a813f06f31bacb3ba 2024-11-12T14:18:04,330 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/9a79c700b9e24b94b51e448df48f5fc7 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/9a79c700b9e24b94b51e448df48f5fc7 2024-11-12T14:18:04,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/18680df5e78a450fac478b388fa0efb4 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/18680df5e78a450fac478b388fa0efb4 2024-11-12T14:18:04,333 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/d56f90a96c464bc1b8f659f3a302d390 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/d56f90a96c464bc1b8f659f3a302d390 2024-11-12T14:18:04,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/7a6791ff583d451a8de6743c050cf228 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/7a6791ff583d451a8de6743c050cf228 2024-11-12T14:18:04,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/25ae97c5bf314e738542b8f7984812ea to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/25ae97c5bf314e738542b8f7984812ea 2024-11-12T14:18:04,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/32d725af0bbb4b25ae8b71e7021d07af, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/1cb3e579dc114cc0bf97244f38a391aa, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/3de2f394235e49f38c731a53c7a7fc50, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/546f5920b5ae467483f868ea5bd3d77d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/519b173fcf234caabd2e8aeb8cb8616e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/e4b5b684dc8e4d799e963c14e1afdc67, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/c2632106b0fa4a5a98a4b8ff5ee4fd21, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/c87fa5ce56ee415c8e6d3995b3b80f21, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/9893efef67204581a5427a998c71b94d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/9e432300f0d24aec9e658f0cff422b0d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/bb68c640b73544b9813e38c582c75f5f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/b27a2b9141bc4b4a88beab9abfc62f7a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/0b652acf7e2544f887f4d08d2bf43808, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/1c0abdf4de8040e38c697331993ac13d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/e09188d2f7904f78a754d2d4fe05ea89, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/6122378601504c898c12f5f21fc45580, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/e25bdf12415c40ae87e510bbe4f0279b, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/7da0380900184affa2df37ac9a06f91f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/05fba06c0208424c99b22cd49d8642e6, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/dfb30d178b5941e093da16f70c13b22d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/ef9d865ddc4a4fe38b287594b99f1857, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/ac2d6061ccb7441aab75e60908eb3139, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/c02934513e6342eb88ad74b54f982321, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/0778cfd69ca44d6dbf6ed5d5d0f4af2e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/bd622497677348afad72b7d1e0b87008, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/efd772221f4f443aba1148fdeb6ade8f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/84e10e08a4b64a30aa960744cb394ce7] to archive 2024-11-12T14:18:04,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T14:18:04,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/32d725af0bbb4b25ae8b71e7021d07af to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/32d725af0bbb4b25ae8b71e7021d07af 2024-11-12T14:18:04,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/1cb3e579dc114cc0bf97244f38a391aa to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/1cb3e579dc114cc0bf97244f38a391aa 2024-11-12T14:18:04,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/3de2f394235e49f38c731a53c7a7fc50 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/3de2f394235e49f38c731a53c7a7fc50 2024-11-12T14:18:04,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/546f5920b5ae467483f868ea5bd3d77d to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/546f5920b5ae467483f868ea5bd3d77d 2024-11-12T14:18:04,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/519b173fcf234caabd2e8aeb8cb8616e to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/519b173fcf234caabd2e8aeb8cb8616e 2024-11-12T14:18:04,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/e4b5b684dc8e4d799e963c14e1afdc67 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/e4b5b684dc8e4d799e963c14e1afdc67 2024-11-12T14:18:04,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/c2632106b0fa4a5a98a4b8ff5ee4fd21 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/c2632106b0fa4a5a98a4b8ff5ee4fd21 2024-11-12T14:18:04,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/c87fa5ce56ee415c8e6d3995b3b80f21 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/c87fa5ce56ee415c8e6d3995b3b80f21 2024-11-12T14:18:04,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/9893efef67204581a5427a998c71b94d to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/9893efef67204581a5427a998c71b94d 2024-11-12T14:18:04,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/9e432300f0d24aec9e658f0cff422b0d to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/9e432300f0d24aec9e658f0cff422b0d 2024-11-12T14:18:04,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/bb68c640b73544b9813e38c582c75f5f to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/bb68c640b73544b9813e38c582c75f5f 2024-11-12T14:18:04,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/b27a2b9141bc4b4a88beab9abfc62f7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/b27a2b9141bc4b4a88beab9abfc62f7a 2024-11-12T14:18:04,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/0b652acf7e2544f887f4d08d2bf43808 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/0b652acf7e2544f887f4d08d2bf43808 2024-11-12T14:18:04,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/1c0abdf4de8040e38c697331993ac13d to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/1c0abdf4de8040e38c697331993ac13d 2024-11-12T14:18:04,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/e09188d2f7904f78a754d2d4fe05ea89 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/e09188d2f7904f78a754d2d4fe05ea89 2024-11-12T14:18:04,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/6122378601504c898c12f5f21fc45580 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/6122378601504c898c12f5f21fc45580 2024-11-12T14:18:04,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/e25bdf12415c40ae87e510bbe4f0279b to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/e25bdf12415c40ae87e510bbe4f0279b 2024-11-12T14:18:04,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/7da0380900184affa2df37ac9a06f91f to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/7da0380900184affa2df37ac9a06f91f 2024-11-12T14:18:04,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/05fba06c0208424c99b22cd49d8642e6 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/05fba06c0208424c99b22cd49d8642e6 2024-11-12T14:18:04,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/dfb30d178b5941e093da16f70c13b22d to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/dfb30d178b5941e093da16f70c13b22d 2024-11-12T14:18:04,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/ef9d865ddc4a4fe38b287594b99f1857 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/ef9d865ddc4a4fe38b287594b99f1857 2024-11-12T14:18:04,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/ac2d6061ccb7441aab75e60908eb3139 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/ac2d6061ccb7441aab75e60908eb3139 2024-11-12T14:18:04,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/c02934513e6342eb88ad74b54f982321 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/c02934513e6342eb88ad74b54f982321 2024-11-12T14:18:04,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/0778cfd69ca44d6dbf6ed5d5d0f4af2e to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/0778cfd69ca44d6dbf6ed5d5d0f4af2e 2024-11-12T14:18:04,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/bd622497677348afad72b7d1e0b87008 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/bd622497677348afad72b7d1e0b87008 2024-11-12T14:18:04,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/efd772221f4f443aba1148fdeb6ade8f to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/efd772221f4f443aba1148fdeb6ade8f 2024-11-12T14:18:04,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/84e10e08a4b64a30aa960744cb394ce7 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/84e10e08a4b64a30aa960744cb394ce7 2024-11-12T14:18:04,407 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/recovered.edits/893.seqid, newMaxSeqId=893, maxSeqId=1 2024-11-12T14:18:04,408 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. 2024-11-12T14:18:04,408 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] regionserver.HRegion(1676): Region close journal for 7a67a89740591358c1149d6b779d0225: Waiting for close lock at 1731421083287Running coprocessor pre-close hooks at 1731421083287Disabling compacts and flushes for region at 1731421083287Disabling writes for close at 1731421083287Obtaining lock to block concurrent updates at 1731421083287Preparing flush snapshotting stores in 7a67a89740591358c1149d6b779d0225 at 1731421083287Finished memstore snapshotting TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=102720, getOffHeapSize=0, getCellsCount=750 at 1731421083288 (+1 ms)Flushing stores of TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225. at 1731421083288Flushing 7a67a89740591358c1149d6b779d0225/A: creating writer at 1731421083288Flushing 7a67a89740591358c1149d6b779d0225/A: appending metadata at 1731421083297 (+9 ms)Flushing 7a67a89740591358c1149d6b779d0225/A: closing flushed file at 1731421083297Flushing 7a67a89740591358c1149d6b779d0225/B: creating writer at 1731421083708 (+411 ms)Flushing 7a67a89740591358c1149d6b779d0225/B: appending metadata at 1731421083714 (+6 ms)Flushing 7a67a89740591358c1149d6b779d0225/B: closing flushed file at 1731421083714Flushing 7a67a89740591358c1149d6b779d0225/C: creating writer at 1731421083733 (+19 ms)Flushing 7a67a89740591358c1149d6b779d0225/C: appending metadata at 1731421083739 (+6 ms)Flushing 7a67a89740591358c1149d6b779d0225/C: closing flushed file at 1731421083739Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2bd1faf2: reopening flushed file at 1731421084169 (+430 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@53281a29: reopening flushed file at 1731421084179 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@40367d6: reopening flushed file at 1731421084191 (+12 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~100.31 KB/102720, currentSize=0 B/0 for 7a67a89740591358c1149d6b779d0225 in 920ms, sequenceid=890, compaction requested=true at 1731421084207 (+16 ms)Writing region close event to WAL at 1731421084387 (+180 ms)Running coprocessor post-close hooks at 1731421084408 (+21 ms)Closed at 1731421084408 2024-11-12T14:18:04,411 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=71}] handler.UnassignRegionHandler(157): Closed 7a67a89740591358c1149d6b779d0225 2024-11-12T14:18:04,411 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=70 updating hbase:meta row=7a67a89740591358c1149d6b779d0225, regionState=CLOSED 2024-11-12T14:18:04,414 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=71, ppid=70, state=RUNNABLE, hasLock=false; CloseRegionProcedure 7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 because future has completed 2024-11-12T14:18:04,420 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-12T14:18:04,420 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; CloseRegionProcedure 7a67a89740591358c1149d6b779d0225, server=2b6d221c5cde,42925,1731420990822 in 1.2810 sec 2024-11-12T14:18:04,422 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=70, resume processing ppid=69 2024-11-12T14:18:04,422 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, ppid=69, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=7a67a89740591358c1149d6b779d0225, UNASSIGN in 1.2900 sec 2024-11-12T14:18:04,425 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-12T14:18:04,425 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.2940 sec 2024-11-12T14:18:04,426 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731421084426"}]},"ts":"1731421084426"} 2024-11-12T14:18:04,427 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T14:18:04,427 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T14:18:04,429 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.3060 sec 2024-11-12T14:18:05,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-12T14:18:05,279 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:18:05,280 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-12T14:18:05,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:18:05,282 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=72, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:18:05,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T14:18:05,282 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=72, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:18:05,284 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225 2024-11-12T14:18:05,286 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A, FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B, FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C, FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/recovered.edits] 2024-11-12T14:18:05,288 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/3429a44812e54c2d88519d70177f6aee to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/3429a44812e54c2d88519d70177f6aee 2024-11-12T14:18:05,289 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/82fdd6cb9bb64a509e4689bc597fe13c to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/82fdd6cb9bb64a509e4689bc597fe13c 2024-11-12T14:18:05,290 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/a04d3dbf6dd747989d529b262a97f239 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/a04d3dbf6dd747989d529b262a97f239 2024-11-12T14:18:05,291 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/bf771136374a4778b76a865f2efb3c17 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/A/bf771136374a4778b76a865f2efb3c17 2024-11-12T14:18:05,294 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/0bd27edda1f54be0b0a702c540704785 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/0bd27edda1f54be0b0a702c540704785 2024-11-12T14:18:05,294 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/5f590a46e7c24202a4cc8cf23056dcc7 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/5f590a46e7c24202a4cc8cf23056dcc7 2024-11-12T14:18:05,295 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/7ef62c7c0fb24f9486841d7b1e63db10 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/7ef62c7c0fb24f9486841d7b1e63db10 2024-11-12T14:18:05,296 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/c6625d1ff25b415eb8a1ed795c0fbb53 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/B/c6625d1ff25b415eb8a1ed795c0fbb53 2024-11-12T14:18:05,298 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/628bc4a61d664691ac4cf9366706d672 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/628bc4a61d664691ac4cf9366706d672 2024-11-12T14:18:05,301 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/6588322c3af341ffa2b3c3a4ec69fe07 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/6588322c3af341ffa2b3c3a4ec69fe07 2024-11-12T14:18:05,302 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/a46abf23f36145b481ae92e152f80cea to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/a46abf23f36145b481ae92e152f80cea 2024-11-12T14:18:05,303 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/bc393662e0d24299b34c1f677923437b to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/C/bc393662e0d24299b34c1f677923437b 2024-11-12T14:18:05,305 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/recovered.edits/893.seqid to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225/recovered.edits/893.seqid 2024-11-12T14:18:05,306 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/7a67a89740591358c1149d6b779d0225 2024-11-12T14:18:05,306 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T14:18:05,308 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=72, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:18:05,310 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T14:18:05,313 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T14:18:05,315 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=72, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:18:05,315 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T14:18:05,315 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731421085315"}]},"ts":"9223372036854775807"} 2024-11-12T14:18:05,317 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T14:18:05,317 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 7a67a89740591358c1149d6b779d0225, NAME => 'TestAcidGuarantees,,1731421057758.7a67a89740591358c1149d6b779d0225.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T14:18:05,317 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T14:18:05,317 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731421085317"}]},"ts":"9223372036854775807"} 2024-11-12T14:18:05,319 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T14:18:05,320 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=72, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:18:05,321 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 40 msec 2024-11-12T14:18:05,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-12T14:18:05,389 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T14:18:05,391 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:18:05,398 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testGetAtomicity Thread=250 (was 248) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1274387348_22 at /127.0.0.1:45582 [Waiting for operation #259] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1315096866_22 at /127.0.0.1:51740 [Waiting for operation #328] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1140 (was 1242), ProcessCount=11 (was 11), AvailableMemoryMB=2563 (was 497) - AvailableMemoryMB LEAK? - 2024-11-12T14:18:05,407 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=250, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=1140, ProcessCount=11, AvailableMemoryMB=2561 2024-11-12T14:18:05,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T14:18:05,409 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T14:18:05,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=73, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T14:18:05,411 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=73, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T14:18:05,411 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:05,411 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 73 2024-11-12T14:18:05,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73 2024-11-12T14:18:05,412 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=73, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T14:18:05,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742138_1314 (size=1061) 2024-11-12T14:18:05,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73 2024-11-12T14:18:05,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73 2024-11-12T14:18:05,822 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3 2024-11-12T14:18:05,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742139_1315 (size=53) 2024-11-12T14:18:06,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73 2024-11-12T14:18:06,228 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:18:06,228 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 2f7e684f010a56c52cbae77227dd8542, disabling compactions & flushes 2024-11-12T14:18:06,228 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:06,228 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:06,228 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. after waiting 0 ms 2024-11-12T14:18:06,228 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:06,228 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:06,228 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 2f7e684f010a56c52cbae77227dd8542: Waiting for close lock at 1731421086228Disabling compacts and flushes for region at 1731421086228Disabling writes for close at 1731421086228Writing region close event to WAL at 1731421086228Closed at 1731421086228 2024-11-12T14:18:06,229 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=73, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T14:18:06,230 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731421086229"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731421086229"}]},"ts":"1731421086229"} 2024-11-12T14:18:06,231 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T14:18:06,232 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=73, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T14:18:06,232 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731421086232"}]},"ts":"1731421086232"} 2024-11-12T14:18:06,234 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T14:18:06,234 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2f7e684f010a56c52cbae77227dd8542, ASSIGN}] 2024-11-12T14:18:06,238 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2f7e684f010a56c52cbae77227dd8542, ASSIGN 2024-11-12T14:18:06,239 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2f7e684f010a56c52cbae77227dd8542, ASSIGN; state=OFFLINE, location=2b6d221c5cde,42925,1731420990822; forceNewPlan=false, retain=false 2024-11-12T14:18:06,389 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=2f7e684f010a56c52cbae77227dd8542, regionState=OPENING, regionLocation=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:06,391 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=74, ppid=73, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2f7e684f010a56c52cbae77227dd8542, ASSIGN because future has completed 2024-11-12T14:18:06,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822}] 2024-11-12T14:18:06,546 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:06,546 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7752): Opening region: {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:18:06,547 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:06,547 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:18:06,547 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7794): checking encryption for 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:06,547 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(7797): checking classloading for 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:06,548 INFO [StoreOpener-2f7e684f010a56c52cbae77227dd8542-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:06,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73 2024-11-12T14:18:06,549 INFO [StoreOpener-2f7e684f010a56c52cbae77227dd8542-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T14:18:06,549 INFO [StoreOpener-2f7e684f010a56c52cbae77227dd8542-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2f7e684f010a56c52cbae77227dd8542 columnFamilyName A 2024-11-12T14:18:06,550 DEBUG [StoreOpener-2f7e684f010a56c52cbae77227dd8542-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:06,550 INFO [StoreOpener-2f7e684f010a56c52cbae77227dd8542-1 {}] regionserver.HStore(327): Store=2f7e684f010a56c52cbae77227dd8542/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:18:06,550 INFO [StoreOpener-2f7e684f010a56c52cbae77227dd8542-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:06,551 INFO [StoreOpener-2f7e684f010a56c52cbae77227dd8542-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T14:18:06,551 INFO [StoreOpener-2f7e684f010a56c52cbae77227dd8542-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2f7e684f010a56c52cbae77227dd8542 columnFamilyName B 2024-11-12T14:18:06,551 DEBUG [StoreOpener-2f7e684f010a56c52cbae77227dd8542-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:06,551 INFO [StoreOpener-2f7e684f010a56c52cbae77227dd8542-1 {}] regionserver.HStore(327): Store=2f7e684f010a56c52cbae77227dd8542/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:18:06,551 INFO [StoreOpener-2f7e684f010a56c52cbae77227dd8542-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:06,552 INFO [StoreOpener-2f7e684f010a56c52cbae77227dd8542-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T14:18:06,552 INFO [StoreOpener-2f7e684f010a56c52cbae77227dd8542-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2f7e684f010a56c52cbae77227dd8542 columnFamilyName C 2024-11-12T14:18:06,552 DEBUG [StoreOpener-2f7e684f010a56c52cbae77227dd8542-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:06,553 INFO [StoreOpener-2f7e684f010a56c52cbae77227dd8542-1 {}] regionserver.HStore(327): Store=2f7e684f010a56c52cbae77227dd8542/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:18:06,553 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:06,553 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1038): replaying wal for 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:06,554 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:06,554 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:06,555 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1048): stopping wal replay for 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:06,555 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1060): Cleaning up temporary data for 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:06,556 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T14:18:06,556 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1093): writing seq id for 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:06,558 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:18:06,558 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1114): Opened 2f7e684f010a56c52cbae77227dd8542; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=67700045, jitterRate=0.008809283375740051}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T14:18:06,558 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:06,559 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegion(1006): Region open journal for 2f7e684f010a56c52cbae77227dd8542: Running coprocessor pre-open hook at 1731421086547Writing region info on filesystem at 1731421086547Initializing all the Stores at 1731421086548 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731421086548Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731421086548Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731421086548Cleaning up temporary data from old regions at 1731421086555 (+7 ms)Running coprocessor post-open hooks at 1731421086559 (+4 ms)Region opened successfully at 1731421086559 2024-11-12T14:18:06,560 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., pid=75, masterSystemTime=1731421086543 2024-11-12T14:18:06,561 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:06,561 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=75}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:06,562 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=74 updating hbase:meta row=2f7e684f010a56c52cbae77227dd8542, regionState=OPEN, openSeqNum=2, regionLocation=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:06,563 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=75, ppid=74, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 because future has completed 2024-11-12T14:18:06,565 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-12T14:18:06,565 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; OpenRegionProcedure 2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 in 172 msec 2024-11-12T14:18:06,567 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=74, resume processing ppid=73 2024-11-12T14:18:06,567 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, ppid=73, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2f7e684f010a56c52cbae77227dd8542, ASSIGN in 331 msec 2024-11-12T14:18:06,568 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=73, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T14:18:06,568 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731421086568"}]},"ts":"1731421086568"} 2024-11-12T14:18:06,569 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T14:18:06,570 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=73, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T14:18:06,571 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1610 sec 2024-11-12T14:18:07,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=73 2024-11-12T14:18:07,559 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:18:07,560 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11851421, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:07,560 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:18:07,560 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:18:07,561 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:18:07,561 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:18:07,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:18:07,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b8b5c7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:07,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:18:07,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:18:07,562 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:07,563 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40024, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:18:07,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6049062f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:07,564 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:18:07,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:18:07,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:18:07,566 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45700, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:18:07,567 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T14:18:07,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 2b6d221c5cde,45591,1731420989939 2024-11-12T14:18:07,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7c1f0ebf 2024-11-12T14:18:07,567 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T14:18:07,568 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40038, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T14:18:07,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T14:18:07,570 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T14:18:07,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-12T14:18:07,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742140_1316 (size=1097) 2024-11-12T14:18:07,981 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-12T14:18:07,981 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-12T14:18:07,984 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T14:18:07,986 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2f7e684f010a56c52cbae77227dd8542, REOPEN/MOVE}] 2024-11-12T14:18:07,987 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2f7e684f010a56c52cbae77227dd8542, REOPEN/MOVE 2024-11-12T14:18:07,988 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=2f7e684f010a56c52cbae77227dd8542, regionState=CLOSING, regionLocation=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:07,989 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2f7e684f010a56c52cbae77227dd8542, REOPEN/MOVE because future has completed 2024-11-12T14:18:07,989 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T14:18:07,990 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822}] 2024-11-12T14:18:08,142 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(122): Close 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:08,142 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T14:18:08,142 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1722): Closing 2f7e684f010a56c52cbae77227dd8542, disabling compactions & flushes 2024-11-12T14:18:08,142 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:08,142 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:08,142 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. after waiting 0 ms 2024-11-12T14:18:08,142 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:08,146 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T14:18:08,147 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:08,147 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegion(1676): Region close journal for 2f7e684f010a56c52cbae77227dd8542: Waiting for close lock at 1731421088142Running coprocessor pre-close hooks at 1731421088142Disabling compacts and flushes for region at 1731421088142Disabling writes for close at 1731421088142Writing region close event to WAL at 1731421088143 (+1 ms)Running coprocessor post-close hooks at 1731421088147 (+4 ms)Closed at 1731421088147 2024-11-12T14:18:08,147 WARN [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] regionserver.HRegionServer(3299): Not adding moved region record: 2f7e684f010a56c52cbae77227dd8542 to self. 2024-11-12T14:18:08,150 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=79}] handler.UnassignRegionHandler(157): Closed 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:08,152 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=2f7e684f010a56c52cbae77227dd8542, regionState=CLOSED 2024-11-12T14:18:08,154 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=79, ppid=78, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 because future has completed 2024-11-12T14:18:08,157 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-12T14:18:08,157 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; CloseRegionProcedure 2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 in 165 msec 2024-11-12T14:18:08,158 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2f7e684f010a56c52cbae77227dd8542, REOPEN/MOVE; state=CLOSED, location=2b6d221c5cde,42925,1731420990822; forceNewPlan=false, retain=true 2024-11-12T14:18:08,309 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=2f7e684f010a56c52cbae77227dd8542, regionState=OPENING, regionLocation=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:08,310 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=78, ppid=77, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2f7e684f010a56c52cbae77227dd8542, REOPEN/MOVE because future has completed 2024-11-12T14:18:08,311 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=80, ppid=78, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822}] 2024-11-12T14:18:08,465 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:08,465 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7752): Opening region: {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:18:08,465 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:08,465 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:18:08,465 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7794): checking encryption for 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:08,465 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(7797): checking classloading for 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:08,466 INFO [StoreOpener-2f7e684f010a56c52cbae77227dd8542-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:08,467 INFO [StoreOpener-2f7e684f010a56c52cbae77227dd8542-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T14:18:08,467 INFO [StoreOpener-2f7e684f010a56c52cbae77227dd8542-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2f7e684f010a56c52cbae77227dd8542 columnFamilyName A 2024-11-12T14:18:08,468 DEBUG [StoreOpener-2f7e684f010a56c52cbae77227dd8542-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:08,468 INFO [StoreOpener-2f7e684f010a56c52cbae77227dd8542-1 {}] regionserver.HStore(327): Store=2f7e684f010a56c52cbae77227dd8542/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:18:08,469 INFO [StoreOpener-2f7e684f010a56c52cbae77227dd8542-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:08,470 INFO [StoreOpener-2f7e684f010a56c52cbae77227dd8542-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T14:18:08,470 INFO [StoreOpener-2f7e684f010a56c52cbae77227dd8542-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2f7e684f010a56c52cbae77227dd8542 columnFamilyName B 2024-11-12T14:18:08,470 DEBUG [StoreOpener-2f7e684f010a56c52cbae77227dd8542-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:08,471 INFO [StoreOpener-2f7e684f010a56c52cbae77227dd8542-1 {}] regionserver.HStore(327): Store=2f7e684f010a56c52cbae77227dd8542/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:18:08,471 INFO [StoreOpener-2f7e684f010a56c52cbae77227dd8542-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:08,472 INFO [StoreOpener-2f7e684f010a56c52cbae77227dd8542-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T14:18:08,472 INFO [StoreOpener-2f7e684f010a56c52cbae77227dd8542-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 2f7e684f010a56c52cbae77227dd8542 columnFamilyName C 2024-11-12T14:18:08,472 DEBUG [StoreOpener-2f7e684f010a56c52cbae77227dd8542-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:08,472 INFO [StoreOpener-2f7e684f010a56c52cbae77227dd8542-1 {}] regionserver.HStore(327): Store=2f7e684f010a56c52cbae77227dd8542/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:18:08,472 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:08,472 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1038): replaying wal for 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:08,473 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:08,474 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:08,475 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1048): stopping wal replay for 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:08,475 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1060): Cleaning up temporary data for 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:08,475 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T14:18:08,476 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1093): writing seq id for 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:08,477 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1114): Opened 2f7e684f010a56c52cbae77227dd8542; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72746720, jitterRate=0.08401060104370117}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T14:18:08,477 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:08,477 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegion(1006): Region open journal for 2f7e684f010a56c52cbae77227dd8542: Running coprocessor pre-open hook at 1731421088465Writing region info on filesystem at 1731421088465Initializing all the Stores at 1731421088466 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731421088466Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731421088466Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731421088466Cleaning up temporary data from old regions at 1731421088475 (+9 ms)Running coprocessor post-open hooks at 1731421088477 (+2 ms)Region opened successfully at 1731421088477 2024-11-12T14:18:08,478 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., pid=80, masterSystemTime=1731421088462 2024-11-12T14:18:08,479 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:08,479 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=80}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:08,480 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=78 updating hbase:meta row=2f7e684f010a56c52cbae77227dd8542, regionState=OPEN, openSeqNum=5, regionLocation=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:08,481 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=80, ppid=78, state=RUNNABLE, hasLock=false; OpenRegionProcedure 2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 because future has completed 2024-11-12T14:18:08,483 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=80, resume processing ppid=78 2024-11-12T14:18:08,483 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, ppid=78, state=SUCCESS, hasLock=false; OpenRegionProcedure 2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 in 170 msec 2024-11-12T14:18:08,484 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=78, resume processing ppid=77 2024-11-12T14:18:08,484 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, ppid=77, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2f7e684f010a56c52cbae77227dd8542, REOPEN/MOVE in 497 msec 2024-11-12T14:18:08,485 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-12T14:18:08,486 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 500 msec 2024-11-12T14:18:08,487 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 916 msec 2024-11-12T14:18:08,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-12T14:18:08,488 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:18:08,488 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@335bd9f4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:08,488 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:18:08,489 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:18:08,489 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:18:08,489 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:18:08,490 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:18:08,490 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@144de3fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:08,490 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:18:08,490 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:18:08,490 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:08,491 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40066, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:18:08,491 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4542927e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:08,492 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@549ff36a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:08,492 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:18:08,492 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:18:08,493 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:18:08,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:18:08,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:18:08,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0895d6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:08,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:18:08,493 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:18:08,494 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:08,494 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40094, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:18:08,495 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5333124c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:08,495 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1495b47, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:08,495 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:18:08,496 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:18:08,496 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:18:08,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:18:08,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:18:08,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5faf8d32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:08,496 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:18:08,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:18:08,497 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:08,497 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40104, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:18:08,498 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b162fde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:08,498 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a145a37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:08,498 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:18:08,498 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:18:08,499 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:18:08,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:18:08,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:18:08,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@57bd8214, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:08,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:18:08,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:18:08,499 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:08,500 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40124, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:18:08,500 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fe4ae7e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:08,501 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7933e7cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:08,501 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:18:08,501 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:18:08,502 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:18:08,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:18:08,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:18:08,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1841320a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:08,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:18:08,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:18:08,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:08,503 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40128, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:18:08,503 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a2d7149, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:08,504 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b124c96, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:08,504 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:18:08,504 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:18:08,505 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:18:08,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:18:08,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:18:08,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c40c23b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:08,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:18:08,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:18:08,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:08,506 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40144, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:18:08,506 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@443e8f5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:08,506 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cd5b9c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:08,506 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:18:08,507 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:18:08,507 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:18:08,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:18:08,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:18:08,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2da8c649, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:08,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:18:08,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:18:08,507 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:08,508 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40164, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:18:08,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41d28ae8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:08,509 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7604174c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:08,509 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:18:08,509 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:18:08,510 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:18:08,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:18:08,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:18:08,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e099a02, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:08,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:18:08,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:18:08,510 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:08,511 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40182, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:18:08,512 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@38abd233, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:08,513 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3df18c7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:08,513 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:18:08,513 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:18:08,514 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:18:08,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:18:08,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:18:08,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7bf1b61c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:08,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:18:08,515 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:18:08,515 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:08,515 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40190, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:18:08,516 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58d10fc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:08,517 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4eb9f257, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:08,517 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:18:08,517 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:18:08,517 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:18:08,517 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:18:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:18:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a2aa907, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:18:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:18:08,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:08,519 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40206, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:18:08,519 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cbcbb09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:08,520 DEBUG [Thread-1456 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:18:08,520 DEBUG [Thread-1460 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:18:08,520 DEBUG [Thread-1471 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:18:08,520 DEBUG [Thread-1458 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:18:08,520 DEBUG [Thread-1469 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:18:08,521 DEBUG [Thread-1454 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:18:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:18:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:18:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:18:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:18:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:18:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:18:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:18:08,521 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:18:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:18:08,522 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:18:08,523 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45712, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:18:08,523 DEBUG [Thread-1473 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:18:08,523 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45710, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:18:08,523 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45722, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:18:08,523 DEBUG [Thread-1454 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:18:08,523 DEBUG [Thread-1467 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:18:08,523 DEBUG [Thread-1454 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:18:08,524 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45714, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:18:08,524 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45724, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:18:08,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:18:08,524 DEBUG [Thread-1462 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:18:08,524 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:18:08,524 DEBUG [Thread-1465 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:18:08,525 DEBUG [Thread-1467 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:18:08,525 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45728, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:18:08,525 DEBUG [Thread-1467 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:18:08,525 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:18:08,525 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45732, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:18:08,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5] 2024-11-12T14:18:08,525 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45746, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:18:08,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=81, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=81, table=TestAcidGuarantees 2024-11-12T14:18:08,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:18:08,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5] 2024-11-12T14:18:08,526 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:18:08,527 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5] 2024-11-12T14:18:08,527 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=81, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=81, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:18:08,527 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5] 2024-11-12T14:18:08,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-12T14:18:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:18:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:18:08,528 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=81, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=81, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:18:08,528 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=82, ppid=81, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:18:08,528 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5] 2024-11-12T14:18:08,529 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5] 2024-11-12T14:18:08,529 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:18:08,529 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45752, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:18:08,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5] 2024-11-12T14:18:08,530 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5] 2024-11-12T14:18:08,533 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5] 2024-11-12T14:18:08,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5] 2024-11-12T14:18:08,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:08,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2f7e684f010a56c52cbae77227dd8542 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T14:18:08,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=A 2024-11-12T14:18:08,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:08,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=B 2024-11-12T14:18:08,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:08,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=C 2024-11-12T14:18:08,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:08,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112f6d2b4160a484d238b350b23238c2082_2f7e684f010a56c52cbae77227dd8542 is 50, key is test_row_0/A:col10/1731421088537/Put/seqid=0 2024-11-12T14:18:08,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:08,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421148582, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:08,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:08,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:08,592 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:08,592 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742141_1317 (size=14594) 2024-11-12T14:18:08,593 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:08,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421148584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:08,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:08,594 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:08,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:08,594 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:08,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421148585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:08,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:08,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:08,594 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:08,601 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:08,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421148592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:08,602 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:08,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:08,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421148593, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:08,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:08,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:08,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:08,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:08,602 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:08,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-12T14:18:08,683 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-11-12T14:18:08,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:08,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:08,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:08,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(58): pid=82 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:08,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:08,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:08,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:08,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421148700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:08,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:08,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:08,705 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:08,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:08,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:08,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421148700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:08,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421148701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:08,706 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:08,707 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:08,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:08,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421148709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:08,714 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:08,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421148710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:08,837 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-11-12T14:18:08,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:08,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:08,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:08,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(58): pid=82 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:08,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:08,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:08,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-12T14:18:08,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:08,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421148912, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:08,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:08,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:08,918 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:08,926 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421148919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:08,926 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:08,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421148919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:08,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:08,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:08,927 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:08,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:08,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:08,927 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:08,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421148919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:08,927 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:08,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:08,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421148919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:08,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:08,927 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:08,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:08,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:08,927 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:08,992 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-11-12T14:18:08,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:08,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:08,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:08,992 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(58): pid=82 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:08,992 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:08,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:08,995 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,006 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112f6d2b4160a484d238b350b23238c2082_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f6d2b4160a484d238b350b23238c2082_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:09,008 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/f49dc62d10244c489e7fd7b93a9ac094, store: [table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:09,008 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/f49dc62d10244c489e7fd7b93a9ac094 is 175, key is test_row_0/A:col10/1731421088537/Put/seqid=0 2024-11-12T14:18:09,042 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742142_1318 (size=39549) 2024-11-12T14:18:09,043 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/f49dc62d10244c489e7fd7b93a9ac094 2024-11-12T14:18:09,095 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/7f8c8f3c77ea45869164b9ae6908d27a is 50, key is test_row_0/B:col10/1731421088537/Put/seqid=0 2024-11-12T14:18:09,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-11-12T14:18:09,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:09,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:09,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:09,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(58): pid=82 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:09,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:09,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742143_1319 (size=12001) 2024-11-12T14:18:09,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:09,156 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/7f8c8f3c77ea45869164b9ae6908d27a 2024-11-12T14:18:09,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-12T14:18:09,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/1435205a24534b4ab4e893415251a1f9 is 50, key is test_row_0/C:col10/1731421088537/Put/seqid=0 2024-11-12T14:18:09,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:09,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421149233, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:09,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:09,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:09,240 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:09,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:09,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421149234, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:09,246 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:09,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421149241, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:09,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:09,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:09,246 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:09,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:09,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:09,246 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:09,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:09,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421149242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:09,252 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:09,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:09,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:09,252 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:09,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421149242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:09,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:09,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:09,255 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:09,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742144_1320 (size=12001) 2024-11-12T14:18:09,294 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/1435205a24534b4ab4e893415251a1f9 2024-11-12T14:18:09,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-11-12T14:18:09,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:09,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:09,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:09,305 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] handler.RSProcedureHandler(58): pid=82 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:09,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=82 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:09,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=82 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:09,307 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/f49dc62d10244c489e7fd7b93a9ac094 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/f49dc62d10244c489e7fd7b93a9ac094 2024-11-12T14:18:09,311 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/f49dc62d10244c489e7fd7b93a9ac094, entries=200, sequenceid=16, filesize=38.6 K 2024-11-12T14:18:09,312 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/7f8c8f3c77ea45869164b9ae6908d27a as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/7f8c8f3c77ea45869164b9ae6908d27a 2024-11-12T14:18:09,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,337 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/7f8c8f3c77ea45869164b9ae6908d27a, entries=150, sequenceid=16, filesize=11.7 K 2024-11-12T14:18:09,338 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/1435205a24534b4ab4e893415251a1f9 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/1435205a24534b4ab4e893415251a1f9 2024-11-12T14:18:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,360 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/1435205a24534b4ab4e893415251a1f9, entries=150, sequenceid=16, filesize=11.7 K 2024-11-12T14:18:09,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,364 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 2f7e684f010a56c52cbae77227dd8542 in 826ms, sequenceid=16, compaction requested=false 2024-11-12T14:18:09,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,364 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-12T14:18:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,365 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=82 2024-11-12T14:18:09,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:09,461 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2902): Flushing 2f7e684f010a56c52cbae77227dd8542 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T14:18:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=A 2024-11-12T14:18:09,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:09,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=B 2024-11-12T14:18:09,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:09,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=C 2024-11-12T14:18:09,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:09,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111222dc12a3d6cc4a028811c0280a78b653_2f7e684f010a56c52cbae77227dd8542 is 50, key is test_row_0/A:col10/1731421088589/Put/seqid=0 2024-11-12T14:18:09,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742145_1321 (size=9714) 2024-11-12T14:18:09,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-12T14:18:09,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:09,789 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:09,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:09,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421149799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:09,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:09,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:09,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:09,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:09,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421149811, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:09,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:09,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:09,824 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:09,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:09,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421149812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:09,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:09,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:09,835 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:09,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:09,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421149819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:09,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:09,837 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:09,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:09,837 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:09,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421149821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:09,837 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:09,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:09,838 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:09,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:09,910 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111222dc12a3d6cc4a028811c0280a78b653_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111222dc12a3d6cc4a028811c0280a78b653_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:09,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/fffd1410429444babacfef9b4ecbd963, store: [table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:09,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/fffd1410429444babacfef9b4ecbd963 is 175, key is test_row_0/A:col10/1731421088589/Put/seqid=0 2024-11-12T14:18:09,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742146_1322 (size=22361) 2024-11-12T14:18:09,928 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=36, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/fffd1410429444babacfef9b4ecbd963 2024-11-12T14:18:09,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:09,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421149929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:09,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:09,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:09,935 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:09,935 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:09,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421149929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:09,935 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:09,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:09,936 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:09,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/b0ba951b5d2444cc8dbee6613760905d is 50, key is test_row_0/B:col10/1731421088589/Put/seqid=0 2024-11-12T14:18:09,947 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:09,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421149939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:09,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:09,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:09,948 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:09,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:09,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421149939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:09,948 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:09,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:09,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:09,949 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:09,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421149940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:09,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:09,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:09,949 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:09,969 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T14:18:09,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742147_1323 (size=9657) 2024-11-12T14:18:09,971 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/b0ba951b5d2444cc8dbee6613760905d 2024-11-12T14:18:09,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/d311f4ef4fa3480695a744a8d0d16a3c is 50, key is test_row_0/C:col10/1731421088589/Put/seqid=0 2024-11-12T14:18:10,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742148_1324 (size=9657) 2024-11-12T14:18:10,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:10,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421150140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:10,146 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:10,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421150140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:10,162 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:10,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421150159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:10,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:10,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,163 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:10,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421150159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:10,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:10,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:10,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421150161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:10,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,163 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:10,408 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=36 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/d311f4ef4fa3480695a744a8d0d16a3c 2024-11-12T14:18:10,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/fffd1410429444babacfef9b4ecbd963 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/fffd1410429444babacfef9b4ecbd963 2024-11-12T14:18:10,459 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/fffd1410429444babacfef9b4ecbd963, entries=100, sequenceid=36, filesize=21.8 K 2024-11-12T14:18:10,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:10,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421150453, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:10,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,460 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:10,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/b0ba951b5d2444cc8dbee6613760905d as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/b0ba951b5d2444cc8dbee6613760905d 2024-11-12T14:18:10,469 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/b0ba951b5d2444cc8dbee6613760905d, entries=100, sequenceid=36, filesize=9.4 K 2024-11-12T14:18:10,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/d311f4ef4fa3480695a744a8d0d16a3c as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/d311f4ef4fa3480695a744a8d0d16a3c 2024-11-12T14:18:10,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:10,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421150461, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:10,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421150472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:10,477 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:10,477 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/d311f4ef4fa3480695a744a8d0d16a3c, entries=100, sequenceid=36, filesize=9.4 K 2024-11-12T14:18:10,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:10,479 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 2f7e684f010a56c52cbae77227dd8542 in 1018ms, sequenceid=36, compaction requested=false 2024-11-12T14:18:10,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.HRegion(2603): Flush status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:10,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:10,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=82}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=82 2024-11-12T14:18:10,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=82 2024-11-12T14:18:10,484 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2f7e684f010a56c52cbae77227dd8542 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T14:18:10,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=A 2024-11-12T14:18:10,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:10,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=B 2024-11-12T14:18:10,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:10,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=C 2024-11-12T14:18:10,485 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:10,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:10,490 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=82, resume processing ppid=81 2024-11-12T14:18:10,490 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, ppid=81, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9520 sec 2024-11-12T14:18:10,494 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=81, table=TestAcidGuarantees in 1.9680 sec 2024-11-12T14:18:10,501 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411124e90c9dc159a4d32a6aaa16982560372_2f7e684f010a56c52cbae77227dd8542 is 50, key is test_row_0/A:col10/1731421090481/Put/seqid=0 2024-11-12T14:18:10,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742149_1325 (size=14594) 2024-11-12T14:18:10,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:10,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421150607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:10,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:10,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421150607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:10,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,624 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:10,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,625 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:10,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=81 2024-11-12T14:18:10,679 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:18:10,679 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:18:10,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=83, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees 2024-11-12T14:18:10,681 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=83, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:18:10,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-12T14:18:10,681 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=83, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:18:10,681 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:18:10,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:10,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421150729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:10,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:10,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421150729, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:10,739 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:10,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,739 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:10,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-12T14:18:10,833 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-12T14:18:10,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:10,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:10,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:10,834 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:10,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:10,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:10,928 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:10,931 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411124e90c9dc159a4d32a6aaa16982560372_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124e90c9dc159a4d32a6aaa16982560372_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:10,932 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/63d8ee2200a04fe09704bf83cc453ac2, store: [table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:10,932 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/63d8ee2200a04fe09704bf83cc453ac2 is 175, key is test_row_0/A:col10/1731421090481/Put/seqid=0 2024-11-12T14:18:10,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742150_1326 (size=39549) 2024-11-12T14:18:10,950 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=49, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/63d8ee2200a04fe09704bf83cc453ac2 2024-11-12T14:18:10,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/5bd897e34f7741beae6731d109793d51 is 50, key is test_row_0/B:col10/1731421090481/Put/seqid=0 2024-11-12T14:18:10,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742151_1327 (size=12001) 2024-11-12T14:18:10,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:10,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421150953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:10,964 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:10,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421150953, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:10,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,967 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:10,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,967 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:10,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:10,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421150969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:10,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,980 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:10,986 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-12T14:18:10,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:10,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:10,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:10,986 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:10,987 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:10,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:10,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:10,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421150990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:10,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:10,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421150991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:10,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:10,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:10,996 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:11,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-12T14:18:11,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-12T14:18:11,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:11,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:11,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:11,141 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:11,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:11,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:11,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:11,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421151270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:11,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:11,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:11,278 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:11,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:11,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421151279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:11,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:11,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:11,287 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:11,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-12T14:18:11,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:11,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:11,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:11,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:11,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:11,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:11,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-12T14:18:11,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/5bd897e34f7741beae6731d109793d51 2024-11-12T14:18:11,370 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/f674fee392e04efc9817bef9fcad98bc is 50, key is test_row_0/C:col10/1731421090481/Put/seqid=0 2024-11-12T14:18:11,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742152_1328 (size=12001) 2024-11-12T14:18:11,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-12T14:18:11,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:11,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:11,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:11,447 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:11,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:11,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:11,600 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-12T14:18:11,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:11,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:11,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:11,600 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:11,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:11,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:11,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-12T14:18:11,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:11,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:11,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:11,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:11,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:11,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:11,774 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=49 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/f674fee392e04efc9817bef9fcad98bc 2024-11-12T14:18:11,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/63d8ee2200a04fe09704bf83cc453ac2 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/63d8ee2200a04fe09704bf83cc453ac2 2024-11-12T14:18:11,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/63d8ee2200a04fe09704bf83cc453ac2, entries=200, sequenceid=49, filesize=38.6 K 2024-11-12T14:18:11,787 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/5bd897e34f7741beae6731d109793d51 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/5bd897e34f7741beae6731d109793d51 2024-11-12T14:18:11,791 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/5bd897e34f7741beae6731d109793d51, entries=150, sequenceid=49, filesize=11.7 K 2024-11-12T14:18:11,792 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/f674fee392e04efc9817bef9fcad98bc as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/f674fee392e04efc9817bef9fcad98bc 2024-11-12T14:18:11,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:11,798 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/f674fee392e04efc9817bef9fcad98bc, entries=150, sequenceid=49, filesize=11.7 K 2024-11-12T14:18:11,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421151789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:11,799 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:11,799 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=107.34 KB/109920 for 2f7e684f010a56c52cbae77227dd8542 in 1315ms, sequenceid=49, compaction requested=true 2024-11-12T14:18:11,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:11,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f7e684f010a56c52cbae77227dd8542:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:18:11,799 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:11,800 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:11,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f7e684f010a56c52cbae77227dd8542:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:18:11,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:11,800 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:11,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f7e684f010a56c52cbae77227dd8542:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:18:11,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:11,800 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:11,800 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 2f7e684f010a56c52cbae77227dd8542/A is initiating minor compaction (all files) 2024-11-12T14:18:11,801 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f7e684f010a56c52cbae77227dd8542/A in TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:11,801 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/f49dc62d10244c489e7fd7b93a9ac094, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/fffd1410429444babacfef9b4ecbd963, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/63d8ee2200a04fe09704bf83cc453ac2] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp, totalSize=99.1 K 2024-11-12T14:18:11,801 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:11,801 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. files: [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/f49dc62d10244c489e7fd7b93a9ac094, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/fffd1410429444babacfef9b4ecbd963, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/63d8ee2200a04fe09704bf83cc453ac2] 2024-11-12T14:18:11,801 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:11,801 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 2f7e684f010a56c52cbae77227dd8542/B is initiating minor compaction (all files) 2024-11-12T14:18:11,801 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f7e684f010a56c52cbae77227dd8542/B in TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:11,801 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting f49dc62d10244c489e7fd7b93a9ac094, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731421088528 2024-11-12T14:18:11,801 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/7f8c8f3c77ea45869164b9ae6908d27a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/b0ba951b5d2444cc8dbee6613760905d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/5bd897e34f7741beae6731d109793d51] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp, totalSize=32.9 K 2024-11-12T14:18:11,801 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting fffd1410429444babacfef9b4ecbd963, keycount=100, bloomtype=ROW, size=21.8 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731421088588 2024-11-12T14:18:11,801 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f8c8f3c77ea45869164b9ae6908d27a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731421088528 2024-11-12T14:18:11,802 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63d8ee2200a04fe09704bf83cc453ac2, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731421089809 2024-11-12T14:18:11,802 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting b0ba951b5d2444cc8dbee6613760905d, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731421088588 2024-11-12T14:18:11,802 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bd897e34f7741beae6731d109793d51, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731421089809 2024-11-12T14:18:11,806 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:11,809 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f7e684f010a56c52cbae77227dd8542#B#compaction#277 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:11,810 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/d95191cc7eab438e91d92ac3b40c4d73 is 50, key is test_row_0/B:col10/1731421090481/Put/seqid=0 2024-11-12T14:18:11,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:11,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2f7e684f010a56c52cbae77227dd8542 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T14:18:11,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=A 2024-11-12T14:18:11,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:11,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=B 2024-11-12T14:18:11,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:11,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=C 2024-11-12T14:18:11,811 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:11,817 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112501607d69ca94e08a7bd6c2fe24b15da_2f7e684f010a56c52cbae77227dd8542 store=[table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:11,819 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112501607d69ca94e08a7bd6c2fe24b15da_2f7e684f010a56c52cbae77227dd8542, store=[table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:11,819 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112501607d69ca94e08a7bd6c2fe24b15da_2f7e684f010a56c52cbae77227dd8542 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:11,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-12T14:18:11,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742153_1329 (size=12104) 2024-11-12T14:18:11,854 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111236113c092ec94eefa36353a88e2dd66c_2f7e684f010a56c52cbae77227dd8542 is 50, key is test_row_0/A:col10/1731421091807/Put/seqid=0 2024-11-12T14:18:11,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742154_1330 (size=4469) 2024-11-12T14:18:11,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742155_1331 (size=14594) 2024-11-12T14:18:11,905 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:11,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-12T14:18:11,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:11,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:11,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:11,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:11,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:11,912 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111236113c092ec94eefa36353a88e2dd66c_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111236113c092ec94eefa36353a88e2dd66c_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:11,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:11,914 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/f5d6681bee3d490497683ea044ad14d7, store: [table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:11,914 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/f5d6681bee3d490497683ea044ad14d7 is 175, key is test_row_0/A:col10/1731421091807/Put/seqid=0 2024-11-12T14:18:11,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:11,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421151922, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:11,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:11,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:11,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:11,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742156_1332 (size=39549) 2024-11-12T14:18:11,965 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=69, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/f5d6681bee3d490497683ea044ad14d7 2024-11-12T14:18:12,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/59496111405049abba90bcf128331ddf is 50, key is test_row_0/B:col10/1731421091807/Put/seqid=0 2024-11-12T14:18:12,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421151999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:12,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:12,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421151999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:12,004 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_1' in 2f7e684f010a56c52cbae77227dd8542 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:18:12,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:12,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:12,005 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:12,005 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_0' in 2f7e684f010a56c52cbae77227dd8542 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:18:12,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:12,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:12,005 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:12,022 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:12,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421152009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:12,023 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_0' in 2f7e684f010a56c52cbae77227dd8542 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:18:12,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:12,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:12,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:12,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742157_1333 (size=12001) 2024-11-12T14:18:12,058 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:12,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421152044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:12,066 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-12T14:18:12,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:12,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:12,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:12,066 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:12,066 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:12,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:12,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:12,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:12,068 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:12,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-12T14:18:12,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:12,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:12,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:12,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:12,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:12,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:12,261 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/d95191cc7eab438e91d92ac3b40c4d73 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/d95191cc7eab438e91d92ac3b40c4d73 2024-11-12T14:18:12,270 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2f7e684f010a56c52cbae77227dd8542/B of 2f7e684f010a56c52cbae77227dd8542 into d95191cc7eab438e91d92ac3b40c4d73(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:12,270 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:12,270 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., storeName=2f7e684f010a56c52cbae77227dd8542/B, priority=13, startTime=1731421091800; duration=0sec 2024-11-12T14:18:12,270 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:12,270 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f7e684f010a56c52cbae77227dd8542:B 2024-11-12T14:18:12,270 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:12,271 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f7e684f010a56c52cbae77227dd8542#A#compaction#276 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:12,272 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/253d58dff539446287d1cc5f7ec6d28c is 175, key is test_row_0/A:col10/1731421090481/Put/seqid=0 2024-11-12T14:18:12,273 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:12,273 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 2f7e684f010a56c52cbae77227dd8542/C is initiating minor compaction (all files) 2024-11-12T14:18:12,273 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f7e684f010a56c52cbae77227dd8542/C in TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:12,273 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/1435205a24534b4ab4e893415251a1f9, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/d311f4ef4fa3480695a744a8d0d16a3c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/f674fee392e04efc9817bef9fcad98bc] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp, totalSize=32.9 K 2024-11-12T14:18:12,273 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 1435205a24534b4ab4e893415251a1f9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731421088528 2024-11-12T14:18:12,274 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting d311f4ef4fa3480695a744a8d0d16a3c, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=36, earliestPutTs=1731421088588 2024-11-12T14:18:12,275 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting f674fee392e04efc9817bef9fcad98bc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731421089809 2024-11-12T14:18:12,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:12,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421152280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:12,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:12,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:12,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:12,291 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f7e684f010a56c52cbae77227dd8542#C#compaction#280 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:12,292 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/1199f00258d840e0aa069ec33588444f is 50, key is test_row_0/C:col10/1731421090481/Put/seqid=0 2024-11-12T14:18:12,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742158_1334 (size=31058) 2024-11-12T14:18:12,333 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/253d58dff539446287d1cc5f7ec6d28c as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/253d58dff539446287d1cc5f7ec6d28c 2024-11-12T14:18:12,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742159_1335 (size=12104) 2024-11-12T14:18:12,345 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2f7e684f010a56c52cbae77227dd8542/A of 2f7e684f010a56c52cbae77227dd8542 into 253d58dff539446287d1cc5f7ec6d28c(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:12,345 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:12,345 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., storeName=2f7e684f010a56c52cbae77227dd8542/A, priority=13, startTime=1731421091799; duration=0sec 2024-11-12T14:18:12,345 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:12,345 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f7e684f010a56c52cbae77227dd8542:A 2024-11-12T14:18:12,347 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/1199f00258d840e0aa069ec33588444f as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/1199f00258d840e0aa069ec33588444f 2024-11-12T14:18:12,366 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2f7e684f010a56c52cbae77227dd8542/C of 2f7e684f010a56c52cbae77227dd8542 into 1199f00258d840e0aa069ec33588444f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:12,367 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:12,367 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., storeName=2f7e684f010a56c52cbae77227dd8542/C, priority=13, startTime=1731421091800; duration=0sec 2024-11-12T14:18:12,367 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:12,367 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f7e684f010a56c52cbae77227dd8542:C 2024-11-12T14:18:12,375 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-12T14:18:12,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:12,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:12,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:12,375 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] handler.RSProcedureHandler(58): pid=84 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:12,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=84 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:12,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=84 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:12,433 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/59496111405049abba90bcf128331ddf 2024-11-12T14:18:12,449 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/476303ec4fab4d57b0f94b9424cd4705 is 50, key is test_row_0/C:col10/1731421091807/Put/seqid=0 2024-11-12T14:18:12,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742160_1336 (size=12001) 2024-11-12T14:18:12,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=69 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/476303ec4fab4d57b0f94b9424cd4705 2024-11-12T14:18:12,501 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/f5d6681bee3d490497683ea044ad14d7 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/f5d6681bee3d490497683ea044ad14d7 2024-11-12T14:18:12,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/f5d6681bee3d490497683ea044ad14d7, entries=200, sequenceid=69, filesize=38.6 K 2024-11-12T14:18:12,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/59496111405049abba90bcf128331ddf as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/59496111405049abba90bcf128331ddf 2024-11-12T14:18:12,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/59496111405049abba90bcf128331ddf, entries=150, sequenceid=69, filesize=11.7 K 2024-11-12T14:18:12,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/476303ec4fab4d57b0f94b9424cd4705 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/476303ec4fab4d57b0f94b9424cd4705 2024-11-12T14:18:12,523 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/476303ec4fab4d57b0f94b9424cd4705, entries=150, sequenceid=69, filesize=11.7 K 2024-11-12T14:18:12,526 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 2f7e684f010a56c52cbae77227dd8542 in 714ms, sequenceid=69, compaction requested=false 2024-11-12T14:18:12,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:12,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=84 2024-11-12T14:18:12,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:12,533 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2902): Flushing 2f7e684f010a56c52cbae77227dd8542 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T14:18:12,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=A 2024-11-12T14:18:12,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:12,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=B 2024-11-12T14:18:12,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:12,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=C 2024-11-12T14:18:12,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:12,562 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411129651440473d94e47909daf8586151122_2f7e684f010a56c52cbae77227dd8542 is 50, key is test_row_0/A:col10/1731421091908/Put/seqid=0 2024-11-12T14:18:12,588 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742161_1337 (size=12154) 2024-11-12T14:18:12,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:12,593 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411129651440473d94e47909daf8586151122_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129651440473d94e47909daf8586151122_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:12,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/7b0b79c81dae49a89d10a2b08be10374, store: [table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:12,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/7b0b79c81dae49a89d10a2b08be10374 is 175, key is test_row_0/A:col10/1731421091908/Put/seqid=0 2024-11-12T14:18:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:12,596 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:12,607 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742162_1338 (size=30955) 2024-11-12T14:18:12,608 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=84, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/7b0b79c81dae49a89d10a2b08be10374 2024-11-12T14:18:12,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/ea818128bd2346ada1e47f2ece07945a is 50, key is test_row_0/B:col10/1731421091908/Put/seqid=0 2024-11-12T14:18:12,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742163_1339 (size=12001) 2024-11-12T14:18:12,648 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/ea818128bd2346ada1e47f2ece07945a 2024-11-12T14:18:12,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/d9db21c9851640f5bfa31a49b4d104af is 50, key is test_row_0/C:col10/1731421091908/Put/seqid=0 2024-11-12T14:18:12,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742164_1340 (size=12001) 2024-11-12T14:18:12,690 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=84 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/d9db21c9851640f5bfa31a49b4d104af 2024-11-12T14:18:12,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/7b0b79c81dae49a89d10a2b08be10374 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/7b0b79c81dae49a89d10a2b08be10374 2024-11-12T14:18:12,712 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/7b0b79c81dae49a89d10a2b08be10374, entries=150, sequenceid=84, filesize=30.2 K 2024-11-12T14:18:12,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/ea818128bd2346ada1e47f2ece07945a as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/ea818128bd2346ada1e47f2ece07945a 2024-11-12T14:18:12,720 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/ea818128bd2346ada1e47f2ece07945a, entries=150, sequenceid=84, filesize=11.7 K 2024-11-12T14:18:12,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/d9db21c9851640f5bfa31a49b4d104af as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/d9db21c9851640f5bfa31a49b4d104af 2024-11-12T14:18:12,730 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/d9db21c9851640f5bfa31a49b4d104af, entries=150, sequenceid=84, filesize=11.7 K 2024-11-12T14:18:12,731 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=93.93 KB/96180 for 2f7e684f010a56c52cbae77227dd8542 in 198ms, sequenceid=84, compaction requested=true 2024-11-12T14:18:12,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.HRegion(2603): Flush status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:12,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:12,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=84}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=84 2024-11-12T14:18:12,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=84 2024-11-12T14:18:12,734 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-12T14:18:12,735 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0520 sec 2024-11-12T14:18:12,736 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=83, table=TestAcidGuarantees in 2.0560 sec 2024-11-12T14:18:12,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:12,748 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2f7e684f010a56c52cbae77227dd8542 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T14:18:12,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=A 2024-11-12T14:18:12,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:12,748 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=B 2024-11-12T14:18:12,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:12,749 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=C 2024-11-12T14:18:12,750 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:12,769 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111297b6a474e61c4c3aba04820dc70ad1f9_2f7e684f010a56c52cbae77227dd8542 is 50, key is test_row_0/A:col10/1731421092723/Put/seqid=0 2024-11-12T14:18:12,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742165_1341 (size=14594) 2024-11-12T14:18:12,801 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:12,809 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111297b6a474e61c4c3aba04820dc70ad1f9_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111297b6a474e61c4c3aba04820dc70ad1f9_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:12,810 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/3fb3491ecf5248ceb5038392f2c6856e, store: [table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:12,811 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/3fb3491ecf5248ceb5038392f2c6856e is 175, key is test_row_0/A:col10/1731421092723/Put/seqid=0 2024-11-12T14:18:12,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742166_1342 (size=39549) 2024-11-12T14:18:12,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=83 2024-11-12T14:18:12,840 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:18:12,840 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:18:12,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees 2024-11-12T14:18:12,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-12T14:18:12,842 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=85, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:18:12,843 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=85, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:18:12,843 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=86, ppid=85, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:18:12,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:12,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421152837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:12,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:12,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:12,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:12,849 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:12,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421152838, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:12,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:12,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:12,850 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:12,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-12T14:18:12,974 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:12,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421152959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:12,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:12,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:12,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:12,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:12,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421152959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:12,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:12,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:12,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:12,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-12T14:18:12,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:12,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:12,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:12,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:12,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:12,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:13,150 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-12T14:18:13,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:13,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:13,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:13,150 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:13,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:13,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:13,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-12T14:18:13,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:13,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421153180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:13,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:13,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:13,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:13,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:13,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421153180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:13,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:13,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:13,186 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:13,226 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=102, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/3fb3491ecf5248ceb5038392f2c6856e 2024-11-12T14:18:13,265 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/f647d15e5c3e4983868f7a9c3eb34a1a is 50, key is test_row_0/B:col10/1731421092723/Put/seqid=0 2024-11-12T14:18:13,305 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-12T14:18:13,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:13,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:13,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:13,306 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:13,306 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:13,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:13,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742167_1343 (size=12001) 2024-11-12T14:18:13,459 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-12T14:18:13,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:13,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:13,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:13,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:13,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:13,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:13,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-12T14:18:13,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:13,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421153489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:13,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:13,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421153489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:13,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:13,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:13,494 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:13,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:13,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:13,494 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:13,614 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-12T14:18:13,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:13,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:13,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:13,614 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:13,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:13,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:13,716 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/f647d15e5c3e4983868f7a9c3eb34a1a 2024-11-12T14:18:13,727 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/a2cbe591cb50452abea901598f5c5aab is 50, key is test_row_0/C:col10/1731421092723/Put/seqid=0 2024-11-12T14:18:13,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742168_1344 (size=12001) 2024-11-12T14:18:13,747 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=102 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/a2cbe591cb50452abea901598f5c5aab 2024-11-12T14:18:13,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/3fb3491ecf5248ceb5038392f2c6856e as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/3fb3491ecf5248ceb5038392f2c6856e 2024-11-12T14:18:13,755 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/3fb3491ecf5248ceb5038392f2c6856e, entries=200, sequenceid=102, filesize=38.6 K 2024-11-12T14:18:13,756 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/f647d15e5c3e4983868f7a9c3eb34a1a as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/f647d15e5c3e4983868f7a9c3eb34a1a 2024-11-12T14:18:13,760 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/f647d15e5c3e4983868f7a9c3eb34a1a, entries=150, sequenceid=102, filesize=11.7 K 2024-11-12T14:18:13,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/a2cbe591cb50452abea901598f5c5aab as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/a2cbe591cb50452abea901598f5c5aab 2024-11-12T14:18:13,767 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-12T14:18:13,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:13,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:13,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:13,768 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/a2cbe591cb50452abea901598f5c5aab, entries=150, sequenceid=102, filesize=11.7 K 2024-11-12T14:18:13,768 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] handler.RSProcedureHandler(58): pid=86 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:13,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=86 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:13,769 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 2f7e684f010a56c52cbae77227dd8542 in 1021ms, sequenceid=102, compaction requested=true 2024-11-12T14:18:13,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=86 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:13,769 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:13,769 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:18:13,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f7e684f010a56c52cbae77227dd8542:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:18:13,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:13,770 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:18:13,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f7e684f010a56c52cbae77227dd8542:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:18:13,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:13,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f7e684f010a56c52cbae77227dd8542:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:18:13,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:13,773 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141111 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:18:13,773 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:18:13,773 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 2f7e684f010a56c52cbae77227dd8542/A is initiating minor compaction (all files) 2024-11-12T14:18:13,773 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 2f7e684f010a56c52cbae77227dd8542/B is initiating minor compaction (all files) 2024-11-12T14:18:13,773 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f7e684f010a56c52cbae77227dd8542/B in TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:13,773 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f7e684f010a56c52cbae77227dd8542/A in TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:13,773 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/253d58dff539446287d1cc5f7ec6d28c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/f5d6681bee3d490497683ea044ad14d7, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/7b0b79c81dae49a89d10a2b08be10374, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/3fb3491ecf5248ceb5038392f2c6856e] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp, totalSize=137.8 K 2024-11-12T14:18:13,773 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/d95191cc7eab438e91d92ac3b40c4d73, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/59496111405049abba90bcf128331ddf, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/ea818128bd2346ada1e47f2ece07945a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/f647d15e5c3e4983868f7a9c3eb34a1a] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp, totalSize=47.0 K 2024-11-12T14:18:13,773 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:13,773 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. files: [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/253d58dff539446287d1cc5f7ec6d28c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/f5d6681bee3d490497683ea044ad14d7, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/7b0b79c81dae49a89d10a2b08be10374, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/3fb3491ecf5248ceb5038392f2c6856e] 2024-11-12T14:18:13,774 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting d95191cc7eab438e91d92ac3b40c4d73, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731421089809 2024-11-12T14:18:13,774 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 253d58dff539446287d1cc5f7ec6d28c, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731421089809 2024-11-12T14:18:13,774 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 59496111405049abba90bcf128331ddf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731421090604 2024-11-12T14:18:13,774 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5d6681bee3d490497683ea044ad14d7, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731421090585 2024-11-12T14:18:13,774 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting ea818128bd2346ada1e47f2ece07945a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731421091880 2024-11-12T14:18:13,774 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b0b79c81dae49a89d10a2b08be10374, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731421091880 2024-11-12T14:18:13,774 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting f647d15e5c3e4983868f7a9c3eb34a1a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731421092674 2024-11-12T14:18:13,775 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fb3491ecf5248ceb5038392f2c6856e, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731421092674 2024-11-12T14:18:13,782 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:13,785 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112d790021ffd13463ca3c6e26d7035840e_2f7e684f010a56c52cbae77227dd8542 store=[table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:13,788 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112d790021ffd13463ca3c6e26d7035840e_2f7e684f010a56c52cbae77227dd8542, store=[table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:13,788 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d790021ffd13463ca3c6e26d7035840e_2f7e684f010a56c52cbae77227dd8542 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:13,791 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f7e684f010a56c52cbae77227dd8542#B#compaction#289 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:13,792 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/776e78ef77864db89eacb08e8daacdb3 is 50, key is test_row_0/B:col10/1731421092723/Put/seqid=0 2024-11-12T14:18:13,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742170_1346 (size=12241) 2024-11-12T14:18:13,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742169_1345 (size=4469) 2024-11-12T14:18:13,800 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/776e78ef77864db89eacb08e8daacdb3 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/776e78ef77864db89eacb08e8daacdb3 2024-11-12T14:18:13,801 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f7e684f010a56c52cbae77227dd8542#A#compaction#288 average throughput is 1.29 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:13,801 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/20975ee9f2f14b04a6dd4a4d1ca89914 is 175, key is test_row_0/A:col10/1731421092723/Put/seqid=0 2024-11-12T14:18:13,806 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742171_1347 (size=31195) 2024-11-12T14:18:13,807 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2f7e684f010a56c52cbae77227dd8542/B of 2f7e684f010a56c52cbae77227dd8542 into 776e78ef77864db89eacb08e8daacdb3(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:13,807 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:13,808 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., storeName=2f7e684f010a56c52cbae77227dd8542/B, priority=12, startTime=1731421093770; duration=0sec 2024-11-12T14:18:13,808 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:13,808 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f7e684f010a56c52cbae77227dd8542:B 2024-11-12T14:18:13,808 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:18:13,812 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:18:13,812 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 2f7e684f010a56c52cbae77227dd8542/C is initiating minor compaction (all files) 2024-11-12T14:18:13,812 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f7e684f010a56c52cbae77227dd8542/C in TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:13,812 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/1199f00258d840e0aa069ec33588444f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/476303ec4fab4d57b0f94b9424cd4705, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/d9db21c9851640f5bfa31a49b4d104af, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/a2cbe591cb50452abea901598f5c5aab] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp, totalSize=47.0 K 2024-11-12T14:18:13,813 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 1199f00258d840e0aa069ec33588444f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=49, earliestPutTs=1731421089809 2024-11-12T14:18:13,813 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 476303ec4fab4d57b0f94b9424cd4705, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=69, earliestPutTs=1731421090604 2024-11-12T14:18:13,813 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/20975ee9f2f14b04a6dd4a4d1ca89914 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/20975ee9f2f14b04a6dd4a4d1ca89914 2024-11-12T14:18:13,816 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting d9db21c9851640f5bfa31a49b4d104af, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=84, earliestPutTs=1731421091880 2024-11-12T14:18:13,816 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting a2cbe591cb50452abea901598f5c5aab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731421092674 2024-11-12T14:18:13,817 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2f7e684f010a56c52cbae77227dd8542/A of 2f7e684f010a56c52cbae77227dd8542 into 20975ee9f2f14b04a6dd4a4d1ca89914(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:13,817 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:13,817 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., storeName=2f7e684f010a56c52cbae77227dd8542/A, priority=12, startTime=1731421093769; duration=0sec 2024-11-12T14:18:13,817 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:13,817 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f7e684f010a56c52cbae77227dd8542:A 2024-11-12T14:18:13,825 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f7e684f010a56c52cbae77227dd8542#C#compaction#290 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:13,825 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/e369bc5424ed4ef7bb5b4258b59e19f4 is 50, key is test_row_0/C:col10/1731421092723/Put/seqid=0 2024-11-12T14:18:13,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742172_1348 (size=12241) 2024-11-12T14:18:13,834 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/e369bc5424ed4ef7bb5b4258b59e19f4 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/e369bc5424ed4ef7bb5b4258b59e19f4 2024-11-12T14:18:13,838 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 2f7e684f010a56c52cbae77227dd8542/C of 2f7e684f010a56c52cbae77227dd8542 into e369bc5424ed4ef7bb5b4258b59e19f4(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:13,838 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:13,838 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., storeName=2f7e684f010a56c52cbae77227dd8542/C, priority=12, startTime=1731421093770; duration=0sec 2024-11-12T14:18:13,838 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:13,838 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f7e684f010a56c52cbae77227dd8542:C 2024-11-12T14:18:13,921 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=86 2024-11-12T14:18:13,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:13,921 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2902): Flushing 2f7e684f010a56c52cbae77227dd8542 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T14:18:13,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=A 2024-11-12T14:18:13,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:13,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=B 2024-11-12T14:18:13,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:13,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=C 2024-11-12T14:18:13,922 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:13,929 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ed8a597c7eeb4ebb80c8947a1569e9f5_2f7e684f010a56c52cbae77227dd8542 is 50, key is test_row_0/A:col10/1731421092836/Put/seqid=0 2024-11-12T14:18:13,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742173_1349 (size=12154) 2024-11-12T14:18:13,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-12T14:18:14,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:14,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:14,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:14,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421154043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:14,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421154047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:14,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:14,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421154057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,060 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:14,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:14,073 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421154060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:14,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421154060, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:14,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,074 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:14,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,074 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:14,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:14,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421154169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:14,178 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:14,178 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421154169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:14,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,179 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:14,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:14,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421154169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:14,179 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:14,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,179 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:14,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:14,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421154179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:14,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:14,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421154179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:14,182 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:14,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:14,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:14,350 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ed8a597c7eeb4ebb80c8947a1569e9f5_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ed8a597c7eeb4ebb80c8947a1569e9f5_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:14,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/5cfa37444c0e4edaabd94ab45160e8c9, store: [table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:14,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/5cfa37444c0e4edaabd94ab45160e8c9 is 175, key is test_row_0/A:col10/1731421092836/Put/seqid=0 2024-11-12T14:18:14,387 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742174_1350 (size=30955) 2024-11-12T14:18:14,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:14,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421154389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:14,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:14,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421154389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:14,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:14,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:14,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421154389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:14,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,395 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:14,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:14,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:14,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421154391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:14,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:14,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421154391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:14,396 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:14,396 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=119, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/5cfa37444c0e4edaabd94ab45160e8c9 2024-11-12T14:18:14,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:14,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/e2400819ef58409b9ed0b4a38c270c90 is 50, key is test_row_0/B:col10/1731421092836/Put/seqid=0 2024-11-12T14:18:14,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742175_1351 (size=12001) 2024-11-12T14:18:14,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:14,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421154699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:14,708 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:14,708 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421154700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:14,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421154700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:14,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421154700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:14,709 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,710 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:14,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421154700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:14,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:14,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:14,843 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/e2400819ef58409b9ed0b4a38c270c90 2024-11-12T14:18:14,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/c2624df9e93c4ca892f4488e17803644 is 50, key is test_row_0/C:col10/1731421092836/Put/seqid=0 2024-11-12T14:18:14,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742176_1352 (size=12001) 2024-11-12T14:18:14,869 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=119 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/c2624df9e93c4ca892f4488e17803644 2024-11-12T14:18:14,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/5cfa37444c0e4edaabd94ab45160e8c9 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/5cfa37444c0e4edaabd94ab45160e8c9 2024-11-12T14:18:14,880 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/5cfa37444c0e4edaabd94ab45160e8c9, entries=150, sequenceid=119, filesize=30.2 K 2024-11-12T14:18:14,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/e2400819ef58409b9ed0b4a38c270c90 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/e2400819ef58409b9ed0b4a38c270c90 2024-11-12T14:18:14,888 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/e2400819ef58409b9ed0b4a38c270c90, entries=150, sequenceid=119, filesize=11.7 K 2024-11-12T14:18:14,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/c2624df9e93c4ca892f4488e17803644 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/c2624df9e93c4ca892f4488e17803644 2024-11-12T14:18:14,894 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/c2624df9e93c4ca892f4488e17803644, entries=150, sequenceid=119, filesize=11.7 K 2024-11-12T14:18:14,895 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 2f7e684f010a56c52cbae77227dd8542 in 974ms, sequenceid=119, compaction requested=false 2024-11-12T14:18:14,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.HRegion(2603): Flush status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:14,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:14,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=86}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=86 2024-11-12T14:18:14,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=86 2024-11-12T14:18:14,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=86, resume processing ppid=85 2024-11-12T14:18:14,898 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, ppid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0540 sec 2024-11-12T14:18:14,899 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=85, table=TestAcidGuarantees in 2.0590 sec 2024-11-12T14:18:14,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=85 2024-11-12T14:18:14,990 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:18:14,990 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:18:14,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees 2024-11-12T14:18:14,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-12T14:18:14,992 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:18:14,993 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=87, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:18:14,993 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:18:15,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-12T14:18:15,146 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=88 2024-11-12T14:18:15,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:15,146 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2902): Flushing 2f7e684f010a56c52cbae77227dd8542 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T14:18:15,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=A 2024-11-12T14:18:15,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:15,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=B 2024-11-12T14:18:15,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:15,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=C 2024-11-12T14:18:15,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:15,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411121b5ec5f7e58640ed941710fd1d259a88_2f7e684f010a56c52cbae77227dd8542 is 50, key is test_row_0/A:col10/1731421094056/Put/seqid=0 2024-11-12T14:18:15,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742177_1353 (size=12304) 2024-11-12T14:18:15,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:15,186 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411121b5ec5f7e58640ed941710fd1d259a88_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121b5ec5f7e58640ed941710fd1d259a88_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:15,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/03a38b3b818548829dae79bce3c8828a, store: [table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:15,188 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/03a38b3b818548829dae79bce3c8828a is 175, key is test_row_0/A:col10/1731421094056/Put/seqid=0 2024-11-12T14:18:15,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742178_1354 (size=31105) 2024-11-12T14:18:15,214 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=137, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/03a38b3b818548829dae79bce3c8828a 2024-11-12T14:18:15,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/ba2f9df65c38482ba9dba7bae1d1bcf8 is 50, key is test_row_0/B:col10/1731421094056/Put/seqid=0 2024-11-12T14:18:15,233 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742179_1355 (size=12151) 2024-11-12T14:18:15,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:15,263 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:15,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:15,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421155281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:15,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:15,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421155281, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:15,295 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:15,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421155284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:15,295 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:15,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:15,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421155285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:15,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:15,295 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:15,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421155291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:15,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,296 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:15,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,296 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:15,296 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:15,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-12T14:18:15,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:15,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421155399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:15,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:15,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421155399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:15,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:15,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421155399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,407 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:15,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421155399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:15,408 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:15,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421155399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:15,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,408 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:15,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:15,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:15,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-12T14:18:15,623 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:15,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421155610, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:15,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:15,624 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421155619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:15,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:15,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,625 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421155619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:15,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:15,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,625 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421155619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:15,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,625 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:15,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,625 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:15,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421155619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:15,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:15,634 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/ba2f9df65c38482ba9dba7bae1d1bcf8 2024-11-12T14:18:15,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/23b53ab49cdb46fcab2cae7cd7b152aa is 50, key is test_row_0/C:col10/1731421094056/Put/seqid=0 2024-11-12T14:18:15,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742180_1356 (size=12151) 2024-11-12T14:18:15,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:15,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421155929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:15,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:15,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421155929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:15,941 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:15,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:15,942 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:15,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421155929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:15,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:15,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421155930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:15,942 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:15,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:15,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:15,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421155930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:15,943 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:16,050 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=137 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/23b53ab49cdb46fcab2cae7cd7b152aa 2024-11-12T14:18:16,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/03a38b3b818548829dae79bce3c8828a as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/03a38b3b818548829dae79bce3c8828a 2024-11-12T14:18:16,059 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/03a38b3b818548829dae79bce3c8828a, entries=150, sequenceid=137, filesize=30.4 K 2024-11-12T14:18:16,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/ba2f9df65c38482ba9dba7bae1d1bcf8 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/ba2f9df65c38482ba9dba7bae1d1bcf8 2024-11-12T14:18:16,063 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/ba2f9df65c38482ba9dba7bae1d1bcf8, entries=150, sequenceid=137, filesize=11.9 K 2024-11-12T14:18:16,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/23b53ab49cdb46fcab2cae7cd7b152aa as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/23b53ab49cdb46fcab2cae7cd7b152aa 2024-11-12T14:18:16,068 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/23b53ab49cdb46fcab2cae7cd7b152aa, entries=150, sequenceid=137, filesize=11.9 K 2024-11-12T14:18:16,069 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 2f7e684f010a56c52cbae77227dd8542 in 922ms, sequenceid=137, compaction requested=true 2024-11-12T14:18:16,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.HRegion(2603): Flush status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:16,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:16,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=88}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=88 2024-11-12T14:18:16,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=88 2024-11-12T14:18:16,071 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-12T14:18:16,072 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0770 sec 2024-11-12T14:18:16,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=87, table=TestAcidGuarantees in 1.0820 sec 2024-11-12T14:18:16,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-12T14:18:16,129 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:18:16,130 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:18:16,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees 2024-11-12T14:18:16,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-12T14:18:16,131 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:18:16,132 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=89, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:18:16,132 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=90, ppid=89, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:18:16,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-12T14:18:16,283 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=90 2024-11-12T14:18:16,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:16,284 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2902): Flushing 2f7e684f010a56c52cbae77227dd8542 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T14:18:16,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=A 2024-11-12T14:18:16,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:16,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=B 2024-11-12T14:18:16,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:16,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=C 2024-11-12T14:18:16,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:16,289 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411120856ddd6a821422e8cb3235d49aa94f1_2f7e684f010a56c52cbae77227dd8542 is 50, key is test_row_0/A:col10/1731421095289/Put/seqid=0 2024-11-12T14:18:16,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742181_1357 (size=12304) 2024-11-12T14:18:16,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:16,298 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411120856ddd6a821422e8cb3235d49aa94f1_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411120856ddd6a821422e8cb3235d49aa94f1_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:16,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/c7b6e6eb9d8c421492cb290281d8efa7, store: [table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:16,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/c7b6e6eb9d8c421492cb290281d8efa7 is 175, key is test_row_0/A:col10/1731421095289/Put/seqid=0 2024-11-12T14:18:16,303 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742182_1358 (size=31105) 2024-11-12T14:18:16,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-12T14:18:16,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:16,457 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:16,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:16,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421156476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:16,492 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:16,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421156477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:16,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:16,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421156478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:16,492 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:16,492 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:16,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421156486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:16,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:16,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:16,493 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:16,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:16,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:16,494 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:16,502 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:16,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421156492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:16,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:16,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:16,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:16,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:16,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421156599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:16,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:16,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421156599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:16,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:16,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421156600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:16,605 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:16,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421156600, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:16,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:16,620 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:16,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421156609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:16,620 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:16,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:16,621 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:16,703 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=151, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/c7b6e6eb9d8c421492cb290281d8efa7 2024-11-12T14:18:16,725 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/3e003a3f3f484a5fa47abdf95c7f0479 is 50, key is test_row_0/B:col10/1731421095289/Put/seqid=0 2024-11-12T14:18:16,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742183_1359 (size=12151) 2024-11-12T14:18:16,730 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/3e003a3f3f484a5fa47abdf95c7f0479 2024-11-12T14:18:16,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/d271cfbd761b44198ec438af42ff7cff is 50, key is test_row_0/C:col10/1731421095289/Put/seqid=0 2024-11-12T14:18:16,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742184_1360 (size=12151) 2024-11-12T14:18:16,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-12T14:18:16,814 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:16,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421156809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:16,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:16,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421156809, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:16,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:16,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:16,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421156810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:16,815 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:16,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421156810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:16,816 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:16,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:16,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421156829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:16,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:16,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:16,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:17,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:17,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421157119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:17,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:17,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:17,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:17,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421157120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:17,129 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:17,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:17,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:17,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:17,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:17,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421157120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:17,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:17,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:17,130 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:17,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:17,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421157129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:17,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:17,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:17,138 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:17,146 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=151 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/d271cfbd761b44198ec438af42ff7cff 2024-11-12T14:18:17,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/c7b6e6eb9d8c421492cb290281d8efa7 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/c7b6e6eb9d8c421492cb290281d8efa7 2024-11-12T14:18:17,154 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/c7b6e6eb9d8c421492cb290281d8efa7, entries=150, sequenceid=151, filesize=30.4 K 2024-11-12T14:18:17,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/3e003a3f3f484a5fa47abdf95c7f0479 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/3e003a3f3f484a5fa47abdf95c7f0479 2024-11-12T14:18:17,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:17,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421157149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:17,157 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:17,161 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/3e003a3f3f484a5fa47abdf95c7f0479, entries=150, sequenceid=151, filesize=11.9 K 2024-11-12T14:18:17,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/d271cfbd761b44198ec438af42ff7cff as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/d271cfbd761b44198ec438af42ff7cff 2024-11-12T14:18:17,165 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/d271cfbd761b44198ec438af42ff7cff, entries=150, sequenceid=151, filesize=11.9 K 2024-11-12T14:18:17,166 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 2f7e684f010a56c52cbae77227dd8542 in 882ms, sequenceid=151, compaction requested=true 2024-11-12T14:18:17,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.HRegion(2603): Flush status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:17,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:17,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=90}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=90 2024-11-12T14:18:17,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=90 2024-11-12T14:18:17,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=90, resume processing ppid=89 2024-11-12T14:18:17,169 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, ppid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0350 sec 2024-11-12T14:18:17,172 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=89, table=TestAcidGuarantees in 1.0400 sec 2024-11-12T14:18:17,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=89 2024-11-12T14:18:17,270 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:18:17,270 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:18:17,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees 2024-11-12T14:18:17,275 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:18:17,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-12T14:18:17,278 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=91, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:18:17,278 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:18:17,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-12T14:18:17,432 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=92 2024-11-12T14:18:17,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:17,432 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2902): Flushing 2f7e684f010a56c52cbae77227dd8542 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T14:18:17,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=A 2024-11-12T14:18:17,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:17,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=B 2024-11-12T14:18:17,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:17,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=C 2024-11-12T14:18:17,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:17,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112f0890af18a2b43d9bcbb70a956dafa13_2f7e684f010a56c52cbae77227dd8542 is 50, key is test_row_0/A:col10/1731421096490/Put/seqid=0 2024-11-12T14:18:17,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742185_1361 (size=12304) 2024-11-12T14:18:17,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-12T14:18:17,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:17,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:17,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:17,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421157667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:17,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:17,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:17,672 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:17,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:17,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421157669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:17,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:17,683 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:17,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:17,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:17,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421157670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:17,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:17,684 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:17,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:17,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421157671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:17,685 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:17,685 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:17,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421157672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:17,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:17,791 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:17,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421157779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:17,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:17,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:17,792 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:17,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:17,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421157789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:17,795 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:17,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:17,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:17,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421157790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:17,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:17,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:17,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:17,796 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:17,796 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:17,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421157790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:17,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:17,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:17,796 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:17,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:17,865 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112f0890af18a2b43d9bcbb70a956dafa13_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f0890af18a2b43d9bcbb70a956dafa13_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:17,866 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/b594768a7bd74f828a17e2ced94beb50, store: [table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:17,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/b594768a7bd74f828a17e2ced94beb50 is 175, key is test_row_0/A:col10/1731421096490/Put/seqid=0 2024-11-12T14:18:17,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742186_1362 (size=31105) 2024-11-12T14:18:17,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-12T14:18:18,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:18,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421158000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:18,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:18,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:18,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421158000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:18,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:18,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:18,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:18,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421158000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:18,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:18,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:18,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:18,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:18,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:18,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:18,011 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:18,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421158000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:18,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:18,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:18,011 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:18,270 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/b594768a7bd74f828a17e2ced94beb50 2024-11-12T14:18:18,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/384c719e98d447e5867175ea75490a31 is 50, key is test_row_0/B:col10/1731421096490/Put/seqid=0 2024-11-12T14:18:18,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742187_1363 (size=12151) 2024-11-12T14:18:18,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421158319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:18,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421158319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:18,327 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:18,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421158319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:18,327 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:18,328 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:18,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421158319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:18,328 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:18,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-12T14:18:18,680 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/384c719e98d447e5867175ea75490a31 2024-11-12T14:18:18,687 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/171aa9e980134e08b6b08d5e10c30c3c is 50, key is test_row_0/C:col10/1731421096490/Put/seqid=0 2024-11-12T14:18:18,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742188_1364 (size=12151) 2024-11-12T14:18:18,705 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/171aa9e980134e08b6b08d5e10c30c3c 2024-11-12T14:18:18,705 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:18,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421158699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:18,706 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_2' in 2f7e684f010a56c52cbae77227dd8542 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:18:18,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:18,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:18,706 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:18,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/b594768a7bd74f828a17e2ced94beb50 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/b594768a7bd74f828a17e2ced94beb50 2024-11-12T14:18:18,728 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/b594768a7bd74f828a17e2ced94beb50, entries=150, sequenceid=169, filesize=30.4 K 2024-11-12T14:18:18,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/384c719e98d447e5867175ea75490a31 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/384c719e98d447e5867175ea75490a31 2024-11-12T14:18:18,736 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/384c719e98d447e5867175ea75490a31, entries=150, sequenceid=169, filesize=11.9 K 2024-11-12T14:18:18,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/171aa9e980134e08b6b08d5e10c30c3c as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/171aa9e980134e08b6b08d5e10c30c3c 2024-11-12T14:18:18,742 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/171aa9e980134e08b6b08d5e10c30c3c, entries=150, sequenceid=169, filesize=11.9 K 2024-11-12T14:18:18,742 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 2f7e684f010a56c52cbae77227dd8542 in 1310ms, sequenceid=169, compaction requested=true 2024-11-12T14:18:18,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.HRegion(2603): Flush status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:18,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:18,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=92}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=92 2024-11-12T14:18:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=92 2024-11-12T14:18:18,746 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-12T14:18:18,746 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4660 sec 2024-11-12T14:18:18,755 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=91, table=TestAcidGuarantees in 1.4760 sec 2024-11-12T14:18:18,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:18,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2f7e684f010a56c52cbae77227dd8542 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T14:18:18,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=A 2024-11-12T14:18:18,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:18,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=B 2024-11-12T14:18:18,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:18,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=C 2024-11-12T14:18:18,847 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:18,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112885af978205e46519d20699715257653_2f7e684f010a56c52cbae77227dd8542 is 50, key is test_row_0/A:col10/1731421097671/Put/seqid=0 2024-11-12T14:18:18,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742189_1365 (size=14794) 2024-11-12T14:18:18,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421158886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:18,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:18,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:18,904 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:18,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:18,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421158890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:18,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:18,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:18,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:18,905 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421158892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:18,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:18,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:18,905 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:18,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:18,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421158904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:18,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:18,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:18,910 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:19,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421159010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:19,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421159010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:19,020 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:19,020 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:19,020 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:19,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:19,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:19,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421159010, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:19,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:19,020 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:19,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:19,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:19,021 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:19,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421159019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:19,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:19,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:19,029 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:19,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:19,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421159229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:19,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:19,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:19,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:19,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421159229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:19,241 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:19,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:19,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:19,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421159229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:19,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:19,241 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:19,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:19,241 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:19,242 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:19,248 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:19,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421159239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:19,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:19,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:19,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:19,269 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:19,272 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112885af978205e46519d20699715257653_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112885af978205e46519d20699715257653_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:19,272 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/950bb442494f4ed38d8249012317c1b5, store: [table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:19,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/950bb442494f4ed38d8249012317c1b5 is 175, key is test_row_0/A:col10/1731421097671/Put/seqid=0 2024-11-12T14:18:19,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742190_1366 (size=39749) 2024-11-12T14:18:19,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-12T14:18:19,419 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:18:19,420 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:18:19,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees 2024-11-12T14:18:19,422 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:18:19,424 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=93, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:18:19,424 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=93, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:18:19,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-12T14:18:19,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-12T14:18:19,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:19,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421159549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:19,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:19,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:19,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:19,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:19,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421159549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:19,563 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:19,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421159550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:19,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:19,564 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:19,564 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:19,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:19,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:19,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:19,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:19,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421159559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:19,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:19,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:19,575 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:19,579 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-12T14:18:19,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:19,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:19,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:19,579 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:19,579 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:19,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:19,683 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=184, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/950bb442494f4ed38d8249012317c1b5 2024-11-12T14:18:19,690 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/6d77d37f545d4674b3e8f98027c1d13f is 50, key is test_row_0/B:col10/1731421097671/Put/seqid=0 2024-11-12T14:18:19,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742191_1367 (size=12151) 2024-11-12T14:18:19,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-12T14:18:19,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:19,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:19,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:19,733 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:19,733 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:19,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:19,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-12T14:18:19,889 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-12T14:18:19,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:19,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:19,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:19,889 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:19,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:19,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:20,042 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-12T14:18:20,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:20,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:20,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:20,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:20,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:20,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:20,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-12T14:18:20,078 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:20,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421160069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:20,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:20,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421160069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:20,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:20,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421160069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:20,079 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:20,094 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/6d77d37f545d4674b3e8f98027c1d13f 2024-11-12T14:18:20,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:20,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421160089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:20,100 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/895c59a422e446d8a556ad7fa45ae918 is 50, key is test_row_0/C:col10/1731421097671/Put/seqid=0 2024-11-12T14:18:20,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:20,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:20,102 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:20,118 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742192_1368 (size=12151) 2024-11-12T14:18:20,195 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-12T14:18:20,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:20,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:20,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:20,195 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:20,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:20,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:20,348 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-12T14:18:20,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:20,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:20,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:20,348 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:20,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:20,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:20,502 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-12T14:18:20,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:20,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:20,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:20,502 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] handler.RSProcedureHandler(58): pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:20,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=94 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:20,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=94 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:20,517 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=184 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/895c59a422e446d8a556ad7fa45ae918 2024-11-12T14:18:20,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/950bb442494f4ed38d8249012317c1b5 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/950bb442494f4ed38d8249012317c1b5 2024-11-12T14:18:20,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/950bb442494f4ed38d8249012317c1b5, entries=200, sequenceid=184, filesize=38.8 K 2024-11-12T14:18:20,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/6d77d37f545d4674b3e8f98027c1d13f as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/6d77d37f545d4674b3e8f98027c1d13f 2024-11-12T14:18:20,534 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/6d77d37f545d4674b3e8f98027c1d13f, entries=150, sequenceid=184, filesize=11.9 K 2024-11-12T14:18:20,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/895c59a422e446d8a556ad7fa45ae918 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/895c59a422e446d8a556ad7fa45ae918 2024-11-12T14:18:20,539 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/895c59a422e446d8a556ad7fa45ae918, entries=150, sequenceid=184, filesize=11.9 K 2024-11-12T14:18:20,540 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 2f7e684f010a56c52cbae77227dd8542 in 1693ms, sequenceid=184, compaction requested=true 2024-11-12T14:18:20,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:20,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f7e684f010a56c52cbae77227dd8542:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:18:20,540 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-12T14:18:20,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:20,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f7e684f010a56c52cbae77227dd8542:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:18:20,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:20,540 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-12T14:18:20,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f7e684f010a56c52cbae77227dd8542:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:18:20,540 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:20,544 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72846 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-12T14:18:20,544 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 195214 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-12T14:18:20,544 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 2f7e684f010a56c52cbae77227dd8542/B is initiating minor compaction (all files) 2024-11-12T14:18:20,544 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 2f7e684f010a56c52cbae77227dd8542/A is initiating minor compaction (all files) 2024-11-12T14:18:20,544 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f7e684f010a56c52cbae77227dd8542/A in TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:20,544 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f7e684f010a56c52cbae77227dd8542/B in TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:20,544 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/776e78ef77864db89eacb08e8daacdb3, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/e2400819ef58409b9ed0b4a38c270c90, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/ba2f9df65c38482ba9dba7bae1d1bcf8, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/3e003a3f3f484a5fa47abdf95c7f0479, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/384c719e98d447e5867175ea75490a31, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/6d77d37f545d4674b3e8f98027c1d13f] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp, totalSize=71.1 K 2024-11-12T14:18:20,544 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/20975ee9f2f14b04a6dd4a4d1ca89914, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/5cfa37444c0e4edaabd94ab45160e8c9, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/03a38b3b818548829dae79bce3c8828a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/c7b6e6eb9d8c421492cb290281d8efa7, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/b594768a7bd74f828a17e2ced94beb50, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/950bb442494f4ed38d8249012317c1b5] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp, totalSize=190.6 K 2024-11-12T14:18:20,544 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=10 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:20,544 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. files: [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/20975ee9f2f14b04a6dd4a4d1ca89914, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/5cfa37444c0e4edaabd94ab45160e8c9, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/03a38b3b818548829dae79bce3c8828a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/c7b6e6eb9d8c421492cb290281d8efa7, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/b594768a7bd74f828a17e2ced94beb50, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/950bb442494f4ed38d8249012317c1b5] 2024-11-12T14:18:20,545 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20975ee9f2f14b04a6dd4a4d1ca89914, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731421092674 2024-11-12T14:18:20,545 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 776e78ef77864db89eacb08e8daacdb3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731421092674 2024-11-12T14:18:20,545 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cfa37444c0e4edaabd94ab45160e8c9, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731421092833 2024-11-12T14:18:20,545 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting e2400819ef58409b9ed0b4a38c270c90, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731421092833 2024-11-12T14:18:20,545 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting ba2f9df65c38482ba9dba7bae1d1bcf8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731421094039 2024-11-12T14:18:20,545 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03a38b3b818548829dae79bce3c8828a, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731421094039 2024-11-12T14:18:20,546 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7b6e6eb9d8c421492cb290281d8efa7, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731421095280 2024-11-12T14:18:20,546 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e003a3f3f484a5fa47abdf95c7f0479, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731421095280 2024-11-12T14:18:20,546 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b594768a7bd74f828a17e2ced94beb50, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731421096475 2024-11-12T14:18:20,546 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 384c719e98d447e5867175ea75490a31, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731421096475 2024-11-12T14:18:20,546 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 950bb442494f4ed38d8249012317c1b5, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731421097669 2024-11-12T14:18:20,547 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d77d37f545d4674b3e8f98027c1d13f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731421097670 2024-11-12T14:18:20,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-12T14:18:20,566 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f7e684f010a56c52cbae77227dd8542#B#compaction#306 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:20,567 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/6c678fec983046cc8279199f8938e3b0 is 50, key is test_row_0/B:col10/1731421097671/Put/seqid=0 2024-11-12T14:18:20,569 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:20,571 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112dbac6f586b4e423c9d93b6f55df61a3b_2f7e684f010a56c52cbae77227dd8542 store=[table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:20,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742193_1369 (size=12595) 2024-11-12T14:18:20,573 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112dbac6f586b4e423c9d93b6f55df61a3b_2f7e684f010a56c52cbae77227dd8542, store=[table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:20,574 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112dbac6f586b4e423c9d93b6f55df61a3b_2f7e684f010a56c52cbae77227dd8542 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:20,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742194_1370 (size=4469) 2024-11-12T14:18:20,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=94 2024-11-12T14:18:20,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:20,655 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2902): Flushing 2f7e684f010a56c52cbae77227dd8542 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T14:18:20,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=A 2024-11-12T14:18:20,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:20,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=B 2024-11-12T14:18:20,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:20,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=C 2024-11-12T14:18:20,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:20,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111229dff9c43c3c4853b1b9f482ad193a91_2f7e684f010a56c52cbae77227dd8542 is 50, key is test_row_0/A:col10/1731421098869/Put/seqid=0 2024-11-12T14:18:20,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742195_1371 (size=12304) 2024-11-12T14:18:20,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:20,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:20,885 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:20,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421160876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:20,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:20,885 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:20,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:20,977 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/6c678fec983046cc8279199f8938e3b0 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/6c678fec983046cc8279199f8938e3b0 2024-11-12T14:18:20,978 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f7e684f010a56c52cbae77227dd8542#A#compaction#307 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:20,978 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/e3e4c53cfc1c408399672f86a0a8e948 is 175, key is test_row_0/A:col10/1731421097671/Put/seqid=0 2024-11-12T14:18:20,982 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 2f7e684f010a56c52cbae77227dd8542/B of 2f7e684f010a56c52cbae77227dd8542 into 6c678fec983046cc8279199f8938e3b0(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:20,982 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:20,982 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., storeName=2f7e684f010a56c52cbae77227dd8542/B, priority=10, startTime=1731421100540; duration=0sec 2024-11-12T14:18:20,982 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:20,983 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f7e684f010a56c52cbae77227dd8542:B 2024-11-12T14:18:20,983 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-12T14:18:20,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742196_1372 (size=31549) 2024-11-12T14:18:20,988 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 72846 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-12T14:18:20,988 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 2f7e684f010a56c52cbae77227dd8542/C is initiating minor compaction (all files) 2024-11-12T14:18:20,988 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f7e684f010a56c52cbae77227dd8542/C in TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:20,988 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/e369bc5424ed4ef7bb5b4258b59e19f4, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/c2624df9e93c4ca892f4488e17803644, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/23b53ab49cdb46fcab2cae7cd7b152aa, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/d271cfbd761b44198ec438af42ff7cff, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/171aa9e980134e08b6b08d5e10c30c3c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/895c59a422e446d8a556ad7fa45ae918] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp, totalSize=71.1 K 2024-11-12T14:18:20,989 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting e369bc5424ed4ef7bb5b4258b59e19f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=102, earliestPutTs=1731421092674 2024-11-12T14:18:20,989 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting c2624df9e93c4ca892f4488e17803644, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=119, earliestPutTs=1731421092833 2024-11-12T14:18:20,990 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 23b53ab49cdb46fcab2cae7cd7b152aa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=137, earliestPutTs=1731421094039 2024-11-12T14:18:20,991 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting d271cfbd761b44198ec438af42ff7cff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=151, earliestPutTs=1731421095280 2024-11-12T14:18:20,994 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 171aa9e980134e08b6b08d5e10c30c3c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731421096475 2024-11-12T14:18:20,994 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 895c59a422e446d8a556ad7fa45ae918, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731421097670 2024-11-12T14:18:20,994 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/e3e4c53cfc1c408399672f86a0a8e948 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/e3e4c53cfc1c408399672f86a0a8e948 2024-11-12T14:18:21,000 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 2f7e684f010a56c52cbae77227dd8542/A of 2f7e684f010a56c52cbae77227dd8542 into e3e4c53cfc1c408399672f86a0a8e948(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:21,000 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:21,000 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., storeName=2f7e684f010a56c52cbae77227dd8542/A, priority=10, startTime=1731421100540; duration=0sec 2024-11-12T14:18:21,000 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:21,000 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f7e684f010a56c52cbae77227dd8542:A 2024-11-12T14:18:21,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:21,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421160990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:21,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:21,005 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:21,006 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:21,012 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f7e684f010a56c52cbae77227dd8542#C#compaction#309 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:21,013 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/8d12fd0a4b3b4e8993a8bf033fbe8573 is 50, key is test_row_0/C:col10/1731421097671/Put/seqid=0 2024-11-12T14:18:21,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742197_1373 (size=12595) 2024-11-12T14:18:21,082 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:21,086 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111229dff9c43c3c4853b1b9f482ad193a91_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111229dff9c43c3c4853b1b9f482ad193a91_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:21,087 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/62a601c754eb44749a63a83fb63784c3, store: [table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:21,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/62a601c754eb44749a63a83fb63784c3 is 175, key is test_row_0/A:col10/1731421098869/Put/seqid=0 2024-11-12T14:18:21,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742198_1374 (size=31105) 2024-11-12T14:18:21,093 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=201, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/62a601c754eb44749a63a83fb63784c3 2024-11-12T14:18:21,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:21,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421161090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:21,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:21,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421161090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:21,099 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:21,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421161090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:21,099 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_2' in 2f7e684f010a56c52cbae77227dd8542 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2209 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:18:21,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:21,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:21,100 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:21,101 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_0' in 2f7e684f010a56c52cbae77227dd8542 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2210 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:18:21,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:21,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:21,102 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:21,102 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_1' in 2f7e684f010a56c52cbae77227dd8542 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2216 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:18:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:21,103 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:21,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/ac6d2777d6de472883b6743f6acbb055 is 50, key is test_row_0/B:col10/1731421098869/Put/seqid=0 2024-11-12T14:18:21,132 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:21,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421161119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:21,133 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_0' in 2f7e684f010a56c52cbae77227dd8542 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2229 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:18:21,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:21,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:21,133 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:21,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742199_1375 (size=12151) 2024-11-12T14:18:21,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:21,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421161211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:21,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:21,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:21,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:21,420 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/8d12fd0a4b3b4e8993a8bf033fbe8573 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/8d12fd0a4b3b4e8993a8bf033fbe8573 2024-11-12T14:18:21,424 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 2f7e684f010a56c52cbae77227dd8542/C of 2f7e684f010a56c52cbae77227dd8542 into 8d12fd0a4b3b4e8993a8bf033fbe8573(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:21,424 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:21,424 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., storeName=2f7e684f010a56c52cbae77227dd8542/C, priority=10, startTime=1731421100540; duration=0sec 2024-11-12T14:18:21,424 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:21,424 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f7e684f010a56c52cbae77227dd8542:C 2024-11-12T14:18:21,536 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/ac6d2777d6de472883b6743f6acbb055 2024-11-12T14:18:21,537 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:21,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421161530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:21,537 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:21,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/78a3c57fc5784a5692ef0f7129b42511 is 50, key is test_row_0/C:col10/1731421098869/Put/seqid=0 2024-11-12T14:18:21,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742200_1376 (size=12151) 2024-11-12T14:18:21,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-12T14:18:21,959 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=201 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/78a3c57fc5784a5692ef0f7129b42511 2024-11-12T14:18:21,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/62a601c754eb44749a63a83fb63784c3 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/62a601c754eb44749a63a83fb63784c3 2024-11-12T14:18:21,968 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/62a601c754eb44749a63a83fb63784c3, entries=150, sequenceid=201, filesize=30.4 K 2024-11-12T14:18:21,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/ac6d2777d6de472883b6743f6acbb055 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/ac6d2777d6de472883b6743f6acbb055 2024-11-12T14:18:21,971 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/ac6d2777d6de472883b6743f6acbb055, entries=150, sequenceid=201, filesize=11.9 K 2024-11-12T14:18:21,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/78a3c57fc5784a5692ef0f7129b42511 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/78a3c57fc5784a5692ef0f7129b42511 2024-11-12T14:18:21,978 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/78a3c57fc5784a5692ef0f7129b42511, entries=150, sequenceid=201, filesize=11.9 K 2024-11-12T14:18:21,979 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 2f7e684f010a56c52cbae77227dd8542 in 1324ms, sequenceid=201, compaction requested=false 2024-11-12T14:18:21,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.HRegion(2603): Flush status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:21,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:21,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=94}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=94 2024-11-12T14:18:21,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=94 2024-11-12T14:18:21,981 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=93 2024-11-12T14:18:21,981 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5560 sec 2024-11-12T14:18:21,983 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=93, table=TestAcidGuarantees in 2.5620 sec 2024-11-12T14:18:22,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:22,063 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2f7e684f010a56c52cbae77227dd8542 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-12T14:18:22,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=A 2024-11-12T14:18:22,063 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:22,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=B 2024-11-12T14:18:22,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:22,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=C 2024-11-12T14:18:22,064 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:22,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411120ce8703550374098b120e582dcf84f24_2f7e684f010a56c52cbae77227dd8542 is 50, key is test_row_0/A:col10/1731421102062/Put/seqid=0 2024-11-12T14:18:22,081 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742201_1377 (size=14794) 2024-11-12T14:18:22,166 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:22,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421162155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:22,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:22,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:22,167 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:22,276 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:22,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421162269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:22,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:22,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:22,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:22,482 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:22,487 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411120ce8703550374098b120e582dcf84f24_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411120ce8703550374098b120e582dcf84f24_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:22,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:22,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421162479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:22,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:22,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:22,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:22,490 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/2022d468f65e47b5accc8225f1d8c8f0, store: [table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:22,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/2022d468f65e47b5accc8225f1d8c8f0 is 175, key is test_row_0/A:col10/1731421102062/Put/seqid=0 2024-11-12T14:18:22,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742202_1378 (size=39749) 2024-11-12T14:18:22,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:22,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421162799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:22,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:22,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:22,808 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:22,898 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=220, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/2022d468f65e47b5accc8225f1d8c8f0 2024-11-12T14:18:22,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/225e10f8bd0249378bd7beb5c51ce9c4 is 50, key is test_row_0/B:col10/1731421102062/Put/seqid=0 2024-11-12T14:18:22,934 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742203_1379 (size=12151) 2024-11-12T14:18:23,128 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:23,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421163111, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:23,129 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_1' in 2f7e684f010a56c52cbae77227dd8542 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:18:23,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:23,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:23,129 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:23,133 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:23,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421163119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:23,133 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_2' in 2f7e684f010a56c52cbae77227dd8542 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:18:23,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:23,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:23,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:23,135 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:23,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421163120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:23,135 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_0' in 2f7e684f010a56c52cbae77227dd8542 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:18:23,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:23,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:23,136 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:23,151 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:23,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421163140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:23,152 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_0' in 2f7e684f010a56c52cbae77227dd8542 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:18:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:23,153 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:23,321 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:23,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421163319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:23,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:23,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:23,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:23,334 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/225e10f8bd0249378bd7beb5c51ce9c4 2024-11-12T14:18:23,341 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/b38f70076126487a99ebc0b9c8539156 is 50, key is test_row_0/C:col10/1731421102062/Put/seqid=0 2024-11-12T14:18:23,347 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742204_1380 (size=12151) 2024-11-12T14:18:23,347 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=220 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/b38f70076126487a99ebc0b9c8539156 2024-11-12T14:18:23,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/2022d468f65e47b5accc8225f1d8c8f0 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/2022d468f65e47b5accc8225f1d8c8f0 2024-11-12T14:18:23,357 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/2022d468f65e47b5accc8225f1d8c8f0, entries=200, sequenceid=220, filesize=38.8 K 2024-11-12T14:18:23,358 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/225e10f8bd0249378bd7beb5c51ce9c4 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/225e10f8bd0249378bd7beb5c51ce9c4 2024-11-12T14:18:23,362 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/225e10f8bd0249378bd7beb5c51ce9c4, entries=150, sequenceid=220, filesize=11.9 K 2024-11-12T14:18:23,364 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/b38f70076126487a99ebc0b9c8539156 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/b38f70076126487a99ebc0b9c8539156 2024-11-12T14:18:23,368 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/b38f70076126487a99ebc0b9c8539156, entries=150, sequenceid=220, filesize=11.9 K 2024-11-12T14:18:23,369 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 2f7e684f010a56c52cbae77227dd8542 in 1306ms, sequenceid=220, compaction requested=true 2024-11-12T14:18:23,369 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:23,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f7e684f010a56c52cbae77227dd8542:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:18:23,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:23,369 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:23,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f7e684f010a56c52cbae77227dd8542:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:18:23,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:23,369 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:23,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f7e684f010a56c52cbae77227dd8542:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:18:23,369 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:23,370 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:23,370 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102403 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:23,370 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 2f7e684f010a56c52cbae77227dd8542/A is initiating minor compaction (all files) 2024-11-12T14:18:23,370 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 2f7e684f010a56c52cbae77227dd8542/B is initiating minor compaction (all files) 2024-11-12T14:18:23,370 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f7e684f010a56c52cbae77227dd8542/A in TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:23,370 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f7e684f010a56c52cbae77227dd8542/B in TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:23,370 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/6c678fec983046cc8279199f8938e3b0, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/ac6d2777d6de472883b6743f6acbb055, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/225e10f8bd0249378bd7beb5c51ce9c4] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp, totalSize=36.0 K 2024-11-12T14:18:23,370 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/e3e4c53cfc1c408399672f86a0a8e948, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/62a601c754eb44749a63a83fb63784c3, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/2022d468f65e47b5accc8225f1d8c8f0] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp, totalSize=100.0 K 2024-11-12T14:18:23,371 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:23,371 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. files: [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/e3e4c53cfc1c408399672f86a0a8e948, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/62a601c754eb44749a63a83fb63784c3, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/2022d468f65e47b5accc8225f1d8c8f0] 2024-11-12T14:18:23,371 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c678fec983046cc8279199f8938e3b0, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731421097670 2024-11-12T14:18:23,371 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3e4c53cfc1c408399672f86a0a8e948, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731421097670 2024-11-12T14:18:23,371 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62a601c754eb44749a63a83fb63784c3, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731421098869 2024-11-12T14:18:23,371 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting ac6d2777d6de472883b6743f6acbb055, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731421098869 2024-11-12T14:18:23,372 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2022d468f65e47b5accc8225f1d8c8f0, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731421100830 2024-11-12T14:18:23,372 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 225e10f8bd0249378bd7beb5c51ce9c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731421100830 2024-11-12T14:18:23,380 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:23,381 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f7e684f010a56c52cbae77227dd8542#B#compaction#315 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:23,382 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/9522709c3e4344b08885a1b6b00b621f is 50, key is test_row_0/B:col10/1731421102062/Put/seqid=0 2024-11-12T14:18:23,385 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112cc0933a1517248b086e05a832dce2b76_2f7e684f010a56c52cbae77227dd8542 store=[table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:23,387 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112cc0933a1517248b086e05a832dce2b76_2f7e684f010a56c52cbae77227dd8542, store=[table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:23,387 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112cc0933a1517248b086e05a832dce2b76_2f7e684f010a56c52cbae77227dd8542 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:23,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742205_1381 (size=12697) 2024-11-12T14:18:23,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742206_1382 (size=4469) 2024-11-12T14:18:23,422 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f7e684f010a56c52cbae77227dd8542#A#compaction#316 average throughput is 0.58 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:23,423 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/213ad4684708483face6f736113e3000 is 175, key is test_row_0/A:col10/1731421102062/Put/seqid=0 2024-11-12T14:18:23,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742207_1383 (size=31651) 2024-11-12T14:18:23,457 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/213ad4684708483face6f736113e3000 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/213ad4684708483face6f736113e3000 2024-11-12T14:18:23,468 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2f7e684f010a56c52cbae77227dd8542/A of 2f7e684f010a56c52cbae77227dd8542 into 213ad4684708483face6f736113e3000(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:23,468 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:23,468 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., storeName=2f7e684f010a56c52cbae77227dd8542/A, priority=13, startTime=1731421103369; duration=0sec 2024-11-12T14:18:23,469 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:23,469 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f7e684f010a56c52cbae77227dd8542:A 2024-11-12T14:18:23,469 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:23,470 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:23,470 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 2f7e684f010a56c52cbae77227dd8542/C is initiating minor compaction (all files) 2024-11-12T14:18:23,470 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f7e684f010a56c52cbae77227dd8542/C in TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:23,470 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/8d12fd0a4b3b4e8993a8bf033fbe8573, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/78a3c57fc5784a5692ef0f7129b42511, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/b38f70076126487a99ebc0b9c8539156] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp, totalSize=36.0 K 2024-11-12T14:18:23,473 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d12fd0a4b3b4e8993a8bf033fbe8573, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=184, earliestPutTs=1731421097670 2024-11-12T14:18:23,474 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78a3c57fc5784a5692ef0f7129b42511, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=201, earliestPutTs=1731421098869 2024-11-12T14:18:23,476 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b38f70076126487a99ebc0b9c8539156, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731421100830 2024-11-12T14:18:23,483 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f7e684f010a56c52cbae77227dd8542#C#compaction#317 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:23,484 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/292c8f111d814b7ab00255e186ac3e84 is 50, key is test_row_0/C:col10/1731421102062/Put/seqid=0 2024-11-12T14:18:23,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742208_1384 (size=12697) 2024-11-12T14:18:23,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=93 2024-11-12T14:18:23,589 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:18:23,590 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:18:23,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-12T14:18:23,592 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:18:23,593 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:18:23,593 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:18:23,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-12T14:18:23,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-12T14:18:23,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-12T14:18:23,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:23,746 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing 2f7e684f010a56c52cbae77227dd8542 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-12T14:18:23,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=A 2024-11-12T14:18:23,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:23,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=B 2024-11-12T14:18:23,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:23,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=C 2024-11-12T14:18:23,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:23,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411128d822346371c4aa3ad409ffed592b254_2f7e684f010a56c52cbae77227dd8542 is 50, key is test_row_0/A:col10/1731421102151/Put/seqid=0 2024-11-12T14:18:23,762 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742209_1385 (size=12304) 2024-11-12T14:18:23,821 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/9522709c3e4344b08885a1b6b00b621f as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/9522709c3e4344b08885a1b6b00b621f 2024-11-12T14:18:23,824 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2f7e684f010a56c52cbae77227dd8542/B of 2f7e684f010a56c52cbae77227dd8542 into 9522709c3e4344b08885a1b6b00b621f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:23,824 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:23,824 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., storeName=2f7e684f010a56c52cbae77227dd8542/B, priority=13, startTime=1731421103369; duration=0sec 2024-11-12T14:18:23,825 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:23,825 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f7e684f010a56c52cbae77227dd8542:B 2024-11-12T14:18:23,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-12T14:18:23,916 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/292c8f111d814b7ab00255e186ac3e84 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/292c8f111d814b7ab00255e186ac3e84 2024-11-12T14:18:23,930 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2f7e684f010a56c52cbae77227dd8542/C of 2f7e684f010a56c52cbae77227dd8542 into 292c8f111d814b7ab00255e186ac3e84(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:23,930 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:23,930 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., storeName=2f7e684f010a56c52cbae77227dd8542/C, priority=13, startTime=1731421103369; duration=0sec 2024-11-12T14:18:23,931 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:23,931 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f7e684f010a56c52cbae77227dd8542:C 2024-11-12T14:18:24,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:24,167 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411128d822346371c4aa3ad409ffed592b254_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411128d822346371c4aa3ad409ffed592b254_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:24,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/12b813b71f6649a59fac2f744d97c520, store: [table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:24,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/12b813b71f6649a59fac2f744d97c520 is 175, key is test_row_0/A:col10/1731421102151/Put/seqid=0 2024-11-12T14:18:24,173 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742210_1386 (size=31105) 2024-11-12T14:18:24,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-12T14:18:24,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:24,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:24,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:24,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421164397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:24,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:24,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:24,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:24,515 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:24,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421164509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:24,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:24,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:24,515 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:24,577 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=237, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/12b813b71f6649a59fac2f744d97c520 2024-11-12T14:18:24,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/80ff9e8bc72b4c4a8b0608956972e0d7 is 50, key is test_row_0/B:col10/1731421102151/Put/seqid=0 2024-11-12T14:18:24,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742211_1387 (size=12151) 2024-11-12T14:18:24,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:24,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421164719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:24,728 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:24,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-12T14:18:24,998 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/80ff9e8bc72b4c4a8b0608956972e0d7 2024-11-12T14:18:25,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/a3f7a73ac2c9400cba844e52c0726634 is 50, key is test_row_0/C:col10/1731421102151/Put/seqid=0 2024-11-12T14:18:25,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742212_1388 (size=12151) 2024-11-12T14:18:25,027 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=237 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/a3f7a73ac2c9400cba844e52c0726634 2024-11-12T14:18:25,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/12b813b71f6649a59fac2f744d97c520 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/12b813b71f6649a59fac2f744d97c520 2024-11-12T14:18:25,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:25,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421165039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:25,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:25,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:25,044 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:25,048 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/12b813b71f6649a59fac2f744d97c520, entries=150, sequenceid=237, filesize=30.4 K 2024-11-12T14:18:25,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/80ff9e8bc72b4c4a8b0608956972e0d7 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/80ff9e8bc72b4c4a8b0608956972e0d7 2024-11-12T14:18:25,052 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/80ff9e8bc72b4c4a8b0608956972e0d7, entries=150, sequenceid=237, filesize=11.9 K 2024-11-12T14:18:25,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/a3f7a73ac2c9400cba844e52c0726634 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/a3f7a73ac2c9400cba844e52c0726634 2024-11-12T14:18:25,058 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/a3f7a73ac2c9400cba844e52c0726634, entries=150, sequenceid=237, filesize=11.9 K 2024-11-12T14:18:25,059 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=87.22 KB/89310 for 2f7e684f010a56c52cbae77227dd8542 in 1313ms, sequenceid=237, compaction requested=false 2024-11-12T14:18:25,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:25,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:25,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-12T14:18:25,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-12T14:18:25,064 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-12T14:18:25,064 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4680 sec 2024-11-12T14:18:25,066 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.4740 sec 2024-11-12T14:18:25,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2f7e684f010a56c52cbae77227dd8542 3/3 column families, dataSize=93.93 KB heapSize=279.66 KB 2024-11-12T14:18:25,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=A 2024-11-12T14:18:25,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:25,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=B 2024-11-12T14:18:25,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:25,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=C 2024-11-12T14:18:25,557 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:25,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:25,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411126d266f617c84460592cffa2022fb1575_2f7e684f010a56c52cbae77227dd8542 is 50, key is test_row_0/A:col10/1731421104395/Put/seqid=0 2024-11-12T14:18:25,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742213_1389 (size=14794) 2024-11-12T14:18:25,574 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:25,578 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411126d266f617c84460592cffa2022fb1575_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126d266f617c84460592cffa2022fb1575_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:25,579 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/7b5d17242f7d4f1cb0f513ddf3770030, store: [table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:25,579 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/7b5d17242f7d4f1cb0f513ddf3770030 is 175, key is test_row_0/A:col10/1731421104395/Put/seqid=0 2024-11-12T14:18:25,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742214_1390 (size=39749) 2024-11-12T14:18:25,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:25,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421165667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:25,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:25,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:25,681 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:25,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-12T14:18:25,740 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:18:25,740 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:18:25,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-12T14:18:25,741 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:18:25,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T14:18:25,742 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:18:25,742 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:18:25,794 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:25,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421165789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:25,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:25,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:25,795 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:25,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T14:18:25,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-12T14:18:25,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:25,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:25,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:25,899 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:25,899 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:25,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:25,986 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=256, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/7b5d17242f7d4f1cb0f513ddf3770030 2024-11-12T14:18:25,999 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/0d2aa89fda0e4facb8a1dbcd10813e3e is 50, key is test_row_0/B:col10/1731421104395/Put/seqid=0 2024-11-12T14:18:26,008 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:26,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421165999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:26,009 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:26,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742215_1391 (size=12151) 2024-11-12T14:18:26,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/0d2aa89fda0e4facb8a1dbcd10813e3e 2024-11-12T14:18:26,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-12T14:18:26,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:26,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:26,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:26,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:26,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:26,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:26,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T14:18:26,074 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/12d9a88187d74186a74e87fa73148d86 is 50, key is test_row_0/C:col10/1731421104395/Put/seqid=0 2024-11-12T14:18:26,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742216_1392 (size=12151) 2024-11-12T14:18:26,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/12d9a88187d74186a74e87fa73148d86 2024-11-12T14:18:26,101 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/7b5d17242f7d4f1cb0f513ddf3770030 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/7b5d17242f7d4f1cb0f513ddf3770030 2024-11-12T14:18:26,105 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/7b5d17242f7d4f1cb0f513ddf3770030, entries=200, sequenceid=256, filesize=38.8 K 2024-11-12T14:18:26,106 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/0d2aa89fda0e4facb8a1dbcd10813e3e as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/0d2aa89fda0e4facb8a1dbcd10813e3e 2024-11-12T14:18:26,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/0d2aa89fda0e4facb8a1dbcd10813e3e, entries=150, sequenceid=256, filesize=11.9 K 2024-11-12T14:18:26,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/12d9a88187d74186a74e87fa73148d86 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/12d9a88187d74186a74e87fa73148d86 2024-11-12T14:18:26,126 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/12d9a88187d74186a74e87fa73148d86, entries=150, sequenceid=256, filesize=11.9 K 2024-11-12T14:18:26,127 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~279.61 KB/286320, currentSize=80.51 KB/82440 for 2f7e684f010a56c52cbae77227dd8542 in 571ms, sequenceid=256, compaction requested=true 2024-11-12T14:18:26,127 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:26,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f7e684f010a56c52cbae77227dd8542:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:18:26,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:26,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f7e684f010a56c52cbae77227dd8542:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:18:26,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T14:18:26,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f7e684f010a56c52cbae77227dd8542:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:18:26,127 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-12T14:18:26,127 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:26,127 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:26,130 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:26,130 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 2f7e684f010a56c52cbae77227dd8542/C is initiating minor compaction (all files) 2024-11-12T14:18:26,130 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f7e684f010a56c52cbae77227dd8542/C in TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:26,130 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/292c8f111d814b7ab00255e186ac3e84, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/a3f7a73ac2c9400cba844e52c0726634, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/12d9a88187d74186a74e87fa73148d86] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp, totalSize=36.1 K 2024-11-12T14:18:26,131 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102505 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:26,131 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 2f7e684f010a56c52cbae77227dd8542/A is initiating minor compaction (all files) 2024-11-12T14:18:26,131 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f7e684f010a56c52cbae77227dd8542/A in TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:26,131 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/213ad4684708483face6f736113e3000, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/12b813b71f6649a59fac2f744d97c520, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/7b5d17242f7d4f1cb0f513ddf3770030] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp, totalSize=100.1 K 2024-11-12T14:18:26,131 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:26,131 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. files: [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/213ad4684708483face6f736113e3000, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/12b813b71f6649a59fac2f744d97c520, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/7b5d17242f7d4f1cb0f513ddf3770030] 2024-11-12T14:18:26,132 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 213ad4684708483face6f736113e3000, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731421100830 2024-11-12T14:18:26,132 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 292c8f111d814b7ab00255e186ac3e84, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731421100830 2024-11-12T14:18:26,135 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12b813b71f6649a59fac2f744d97c520, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731421102137 2024-11-12T14:18:26,135 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting a3f7a73ac2c9400cba844e52c0726634, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731421102137 2024-11-12T14:18:26,135 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 12d9a88187d74186a74e87fa73148d86, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731421104374 2024-11-12T14:18:26,135 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b5d17242f7d4f1cb0f513ddf3770030, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731421104354 2024-11-12T14:18:26,145 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f7e684f010a56c52cbae77227dd8542#C#compaction#324 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:26,146 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/0913550b6fd641a5a0bef601c737d28c is 50, key is test_row_0/C:col10/1731421104395/Put/seqid=0 2024-11-12T14:18:26,147 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:26,161 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411122930a70162e046a899fb5e08d326419d_2f7e684f010a56c52cbae77227dd8542 store=[table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:26,163 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411122930a70162e046a899fb5e08d326419d_2f7e684f010a56c52cbae77227dd8542, store=[table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:26,163 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122930a70162e046a899fb5e08d326419d_2f7e684f010a56c52cbae77227dd8542 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:26,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742217_1393 (size=12799) 2024-11-12T14:18:26,179 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/0913550b6fd641a5a0bef601c737d28c as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/0913550b6fd641a5a0bef601c737d28c 2024-11-12T14:18:26,188 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2f7e684f010a56c52cbae77227dd8542/C of 2f7e684f010a56c52cbae77227dd8542 into 0913550b6fd641a5a0bef601c737d28c(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:26,188 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:26,188 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., storeName=2f7e684f010a56c52cbae77227dd8542/C, priority=13, startTime=1731421106127; duration=0sec 2024-11-12T14:18:26,189 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:26,189 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f7e684f010a56c52cbae77227dd8542:C 2024-11-12T14:18:26,189 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:26,190 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36999 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:26,190 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 2f7e684f010a56c52cbae77227dd8542/B is initiating minor compaction (all files) 2024-11-12T14:18:26,190 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f7e684f010a56c52cbae77227dd8542/B in TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:26,190 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/9522709c3e4344b08885a1b6b00b621f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/80ff9e8bc72b4c4a8b0608956972e0d7, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/0d2aa89fda0e4facb8a1dbcd10813e3e] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp, totalSize=36.1 K 2024-11-12T14:18:26,191 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 9522709c3e4344b08885a1b6b00b621f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=220, earliestPutTs=1731421100830 2024-11-12T14:18:26,191 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 80ff9e8bc72b4c4a8b0608956972e0d7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=237, earliestPutTs=1731421102137 2024-11-12T14:18:26,192 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d2aa89fda0e4facb8a1dbcd10813e3e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731421104374 2024-11-12T14:18:26,206 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-12T14:18:26,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:26,206 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing 2f7e684f010a56c52cbae77227dd8542 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T14:18:26,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=A 2024-11-12T14:18:26,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:26,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=B 2024-11-12T14:18:26,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:26,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=C 2024-11-12T14:18:26,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:26,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742218_1394 (size=4469) 2024-11-12T14:18:26,219 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f7e684f010a56c52cbae77227dd8542#B#compaction#326 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:26,219 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/9b5c0a69156c4c548a5c7cf7129615d2 is 50, key is test_row_0/B:col10/1731421104395/Put/seqid=0 2024-11-12T14:18:26,247 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111272211b325be14074a5e9488adce53e40_2f7e684f010a56c52cbae77227dd8542 is 50, key is test_row_0/A:col10/1731421105652/Put/seqid=0 2024-11-12T14:18:26,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742219_1395 (size=12799) 2024-11-12T14:18:26,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742220_1396 (size=12454) 2024-11-12T14:18:26,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:26,266 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111272211b325be14074a5e9488adce53e40_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111272211b325be14074a5e9488adce53e40_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:26,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/93c66b6db996477dbdfe3fed0a7864ca, store: [table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:26,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/93c66b6db996477dbdfe3fed0a7864ca is 175, key is test_row_0/A:col10/1731421105652/Put/seqid=0 2024-11-12T14:18:26,295 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742221_1397 (size=31255) 2024-11-12T14:18:26,296 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/93c66b6db996477dbdfe3fed0a7864ca 2024-11-12T14:18:26,312 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/b575908558044c6a9caf4b80c8c411c7 is 50, key is test_row_0/B:col10/1731421105652/Put/seqid=0 2024-11-12T14:18:26,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742222_1398 (size=12301) 2024-11-12T14:18:26,326 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/b575908558044c6a9caf4b80c8c411c7 2024-11-12T14:18:26,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:26,331 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:26,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/53e63b1f765d4dcb8538e09443a664f4 is 50, key is test_row_0/C:col10/1731421105652/Put/seqid=0 2024-11-12T14:18:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T14:18:26,370 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742223_1399 (size=12301) 2024-11-12T14:18:26,477 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421166465, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:26,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:26,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:26,478 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:26,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:26,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421166590, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:26,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:26,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:26,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:26,610 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f7e684f010a56c52cbae77227dd8542#A#compaction#325 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:26,610 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/b183723aee7149dab7617fb7e5aabfdc is 175, key is test_row_0/A:col10/1731421104395/Put/seqid=0 2024-11-12T14:18:26,614 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742224_1400 (size=31753) 2024-11-12T14:18:26,618 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/b183723aee7149dab7617fb7e5aabfdc as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/b183723aee7149dab7617fb7e5aabfdc 2024-11-12T14:18:26,625 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2f7e684f010a56c52cbae77227dd8542/A of 2f7e684f010a56c52cbae77227dd8542 into b183723aee7149dab7617fb7e5aabfdc(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:26,625 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:26,625 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., storeName=2f7e684f010a56c52cbae77227dd8542/A, priority=13, startTime=1731421106127; duration=0sec 2024-11-12T14:18:26,625 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:26,625 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f7e684f010a56c52cbae77227dd8542:A 2024-11-12T14:18:26,652 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/9b5c0a69156c4c548a5c7cf7129615d2 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/9b5c0a69156c4c548a5c7cf7129615d2 2024-11-12T14:18:26,660 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2f7e684f010a56c52cbae77227dd8542/B of 2f7e684f010a56c52cbae77227dd8542 into 9b5c0a69156c4c548a5c7cf7129615d2(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:26,660 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:26,660 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., storeName=2f7e684f010a56c52cbae77227dd8542/B, priority=13, startTime=1731421106127; duration=0sec 2024-11-12T14:18:26,660 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:26,660 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f7e684f010a56c52cbae77227dd8542:B 2024-11-12T14:18:26,772 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/53e63b1f765d4dcb8538e09443a664f4 2024-11-12T14:18:26,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/93c66b6db996477dbdfe3fed0a7864ca as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/93c66b6db996477dbdfe3fed0a7864ca 2024-11-12T14:18:26,784 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/93c66b6db996477dbdfe3fed0a7864ca, entries=150, sequenceid=272, filesize=30.5 K 2024-11-12T14:18:26,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/b575908558044c6a9caf4b80c8c411c7 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/b575908558044c6a9caf4b80c8c411c7 2024-11-12T14:18:26,788 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/b575908558044c6a9caf4b80c8c411c7, entries=150, sequenceid=272, filesize=12.0 K 2024-11-12T14:18:26,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/53e63b1f765d4dcb8538e09443a664f4 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/53e63b1f765d4dcb8538e09443a664f4 2024-11-12T14:18:26,794 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/53e63b1f765d4dcb8538e09443a664f4, entries=150, sequenceid=272, filesize=12.0 K 2024-11-12T14:18:26,795 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for 2f7e684f010a56c52cbae77227dd8542 in 589ms, sequenceid=272, compaction requested=false 2024-11-12T14:18:26,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:26,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:26,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-12T14:18:26,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-12T14:18:26,799 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-12T14:18:26,799 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0550 sec 2024-11-12T14:18:26,803 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 1.0620 sec 2024-11-12T14:18:26,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:26,811 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2f7e684f010a56c52cbae77227dd8542 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T14:18:26,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=A 2024-11-12T14:18:26,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:26,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=B 2024-11-12T14:18:26,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:26,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=C 2024-11-12T14:18:26,812 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:26,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122024926627e54f6fbb9dd1333eb30931_2f7e684f010a56c52cbae77227dd8542 is 50, key is test_row_0/A:col10/1731421106810/Put/seqid=0 2024-11-12T14:18:26,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742225_1401 (size=12454) 2024-11-12T14:18:26,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-12T14:18:26,879 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:18:26,880 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:18:26,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-12T14:18:26,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T14:18:26,890 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:18:26,891 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:18:26,891 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:18:26,925 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:26,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421166921, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:26,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:26,925 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:26,926 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:26,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T14:18:27,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:27,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421167029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:27,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:27,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:27,041 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:27,043 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T14:18:27,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:27,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:27,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:27,043 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:27,043 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:27,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:27,156 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:27,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45728 deadline: 1731421167149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:27,156 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_0' in 2f7e684f010a56c52cbae77227dd8542 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8264 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:18:27,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:27,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:27,157 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:27,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:27,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45710 deadline: 1731421167159, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:27,167 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_1' in 2f7e684f010a56c52cbae77227dd8542 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8281 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:18:27,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:27,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:27,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:27,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:27,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45752 deadline: 1731421167179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:27,187 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_2' in 2f7e684f010a56c52cbae77227dd8542 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8296 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:18:27,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:27,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:27,187 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:27,198 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:27,198 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T14:18:27,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45722 deadline: 1731421167189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:27,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:27,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:27,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:27,199 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:27,199 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:27,198 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_0' in 2f7e684f010a56c52cbae77227dd8542 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8294 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:18:27,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:27,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:27,199 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:27,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T14:18:27,231 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:27,235 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122024926627e54f6fbb9dd1333eb30931_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122024926627e54f6fbb9dd1333eb30931_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:27,235 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/f1eaea4991834e1d89e18a0ff59230aa, store: [table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:27,236 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/f1eaea4991834e1d89e18a0ff59230aa is 175, key is test_row_0/A:col10/1731421106810/Put/seqid=0 2024-11-12T14:18:27,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:27,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421167249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:27,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742226_1402 (size=31255) 2024-11-12T14:18:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:27,262 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/f1eaea4991834e1d89e18a0ff59230aa 2024-11-12T14:18:27,270 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/4e24c110e5884c1faafc7b3ed82c027d is 50, key is test_row_0/B:col10/1731421106810/Put/seqid=0 2024-11-12T14:18:27,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742227_1403 (size=12301) 2024-11-12T14:18:27,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T14:18:27,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:27,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:27,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:27,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:27,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:27,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:27,506 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T14:18:27,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:27,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:27,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:27,506 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:27,506 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:27,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:27,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T14:18:27,575 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:27,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421167569, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:27,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:27,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:27,575 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:27,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T14:18:27,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:27,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:27,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:27,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:27,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:27,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:27,705 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/4e24c110e5884c1faafc7b3ed82c027d 2024-11-12T14:18:27,711 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/e6d7ea3104b846c0beb193063da09d82 is 50, key is test_row_0/C:col10/1731421106810/Put/seqid=0 2024-11-12T14:18:27,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742228_1404 (size=12301) 2024-11-12T14:18:27,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/e6d7ea3104b846c0beb193063da09d82 2024-11-12T14:18:27,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/f1eaea4991834e1d89e18a0ff59230aa as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/f1eaea4991834e1d89e18a0ff59230aa 2024-11-12T14:18:27,728 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/f1eaea4991834e1d89e18a0ff59230aa, entries=150, sequenceid=292, filesize=30.5 K 2024-11-12T14:18:27,729 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/4e24c110e5884c1faafc7b3ed82c027d as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/4e24c110e5884c1faafc7b3ed82c027d 2024-11-12T14:18:27,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/4e24c110e5884c1faafc7b3ed82c027d, entries=150, sequenceid=292, filesize=12.0 K 2024-11-12T14:18:27,732 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/e6d7ea3104b846c0beb193063da09d82 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/e6d7ea3104b846c0beb193063da09d82 2024-11-12T14:18:27,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/e6d7ea3104b846c0beb193063da09d82, entries=150, sequenceid=292, filesize=12.0 K 2024-11-12T14:18:27,736 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 2f7e684f010a56c52cbae77227dd8542 in 925ms, sequenceid=292, compaction requested=true 2024-11-12T14:18:27,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:27,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f7e684f010a56c52cbae77227dd8542:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:18:27,736 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:27,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:27,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f7e684f010a56c52cbae77227dd8542:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:18:27,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:27,736 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:27,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f7e684f010a56c52cbae77227dd8542:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:18:27,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:27,737 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94263 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:27,737 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 2f7e684f010a56c52cbae77227dd8542/A is initiating minor compaction (all files) 2024-11-12T14:18:27,737 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:27,737 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f7e684f010a56c52cbae77227dd8542/A in TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:27,737 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 2f7e684f010a56c52cbae77227dd8542/B is initiating minor compaction (all files) 2024-11-12T14:18:27,737 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f7e684f010a56c52cbae77227dd8542/B in TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:27,737 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/b183723aee7149dab7617fb7e5aabfdc, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/93c66b6db996477dbdfe3fed0a7864ca, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/f1eaea4991834e1d89e18a0ff59230aa] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp, totalSize=92.1 K 2024-11-12T14:18:27,737 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:27,737 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/9b5c0a69156c4c548a5c7cf7129615d2, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/b575908558044c6a9caf4b80c8c411c7, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/4e24c110e5884c1faafc7b3ed82c027d] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp, totalSize=36.5 K 2024-11-12T14:18:27,737 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. files: [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/b183723aee7149dab7617fb7e5aabfdc, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/93c66b6db996477dbdfe3fed0a7864ca, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/f1eaea4991834e1d89e18a0ff59230aa] 2024-11-12T14:18:27,738 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b183723aee7149dab7617fb7e5aabfdc, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731421104374 2024-11-12T14:18:27,738 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b5c0a69156c4c548a5c7cf7129615d2, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731421104374 2024-11-12T14:18:27,738 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting b575908558044c6a9caf4b80c8c411c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731421105641 2024-11-12T14:18:27,738 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 93c66b6db996477dbdfe3fed0a7864ca, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731421105641 2024-11-12T14:18:27,739 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting f1eaea4991834e1d89e18a0ff59230aa, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731421106442 2024-11-12T14:18:27,739 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e24c110e5884c1faafc7b3ed82c027d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731421106442 2024-11-12T14:18:27,743 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:27,746 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f7e684f010a56c52cbae77227dd8542#B#compaction#334 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:27,746 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/bafd32d69416450f8afeadd9a7be0c18 is 50, key is test_row_0/B:col10/1731421106810/Put/seqid=0 2024-11-12T14:18:27,747 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111264455ffbe4af4e219a7e1283a0711b00_2f7e684f010a56c52cbae77227dd8542 store=[table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:27,749 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111264455ffbe4af4e219a7e1283a0711b00_2f7e684f010a56c52cbae77227dd8542, store=[table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:27,749 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111264455ffbe4af4e219a7e1283a0711b00_2f7e684f010a56c52cbae77227dd8542 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:27,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742229_1405 (size=13051) 2024-11-12T14:18:27,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742230_1406 (size=4469) 2024-11-12T14:18:27,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-12T14:18:27,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:27,812 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing 2f7e684f010a56c52cbae77227dd8542 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T14:18:27,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=A 2024-11-12T14:18:27,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:27,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=B 2024-11-12T14:18:27,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:27,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=C 2024-11-12T14:18:27,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:27,819 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411121541bb0ec5194156aeec62d8505e65fb_2f7e684f010a56c52cbae77227dd8542 is 50, key is test_row_0/A:col10/1731421106900/Put/seqid=0 2024-11-12T14:18:27,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742231_1407 (size=12454) 2024-11-12T14:18:28,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T14:18:28,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:28,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. as already flushing 2024-11-12T14:18:28,156 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/bafd32d69416450f8afeadd9a7be0c18 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/bafd32d69416450f8afeadd9a7be0c18 2024-11-12T14:18:28,160 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2f7e684f010a56c52cbae77227dd8542/B of 2f7e684f010a56c52cbae77227dd8542 into bafd32d69416450f8afeadd9a7be0c18(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:28,160 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:28,161 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., storeName=2f7e684f010a56c52cbae77227dd8542/B, priority=13, startTime=1731421107736; duration=0sec 2024-11-12T14:18:28,161 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:28,161 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f7e684f010a56c52cbae77227dd8542:B 2024-11-12T14:18:28,161 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:28,162 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:28,162 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 2f7e684f010a56c52cbae77227dd8542/C is initiating minor compaction (all files) 2024-11-12T14:18:28,162 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f7e684f010a56c52cbae77227dd8542/C in TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:28,162 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f7e684f010a56c52cbae77227dd8542#A#compaction#333 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:28,162 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/0913550b6fd641a5a0bef601c737d28c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/53e63b1f765d4dcb8538e09443a664f4, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/e6d7ea3104b846c0beb193063da09d82] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp, totalSize=36.5 K 2024-11-12T14:18:28,162 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 0913550b6fd641a5a0bef601c737d28c, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1731421104374 2024-11-12T14:18:28,163 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/8326d065656d4df18176dcf1cf68babb is 175, key is test_row_0/A:col10/1731421106810/Put/seqid=0 2024-11-12T14:18:28,163 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 53e63b1f765d4dcb8538e09443a664f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731421105641 2024-11-12T14:18:28,163 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting e6d7ea3104b846c0beb193063da09d82, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731421106442 2024-11-12T14:18:28,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742232_1408 (size=32005) 2024-11-12T14:18:28,170 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f7e684f010a56c52cbae77227dd8542#C#compaction#336 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:28,171 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/7e1e8b7836e34100b5a5f3afeff57277 is 50, key is test_row_0/C:col10/1731421106810/Put/seqid=0 2024-11-12T14:18:28,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742233_1409 (size=13051) 2024-11-12T14:18:28,179 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/7e1e8b7836e34100b5a5f3afeff57277 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/7e1e8b7836e34100b5a5f3afeff57277 2024-11-12T14:18:28,184 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2f7e684f010a56c52cbae77227dd8542/C of 2f7e684f010a56c52cbae77227dd8542 into 7e1e8b7836e34100b5a5f3afeff57277(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:28,184 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:28,184 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., storeName=2f7e684f010a56c52cbae77227dd8542/C, priority=13, startTime=1731421107736; duration=0sec 2024-11-12T14:18:28,184 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:28,184 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f7e684f010a56c52cbae77227dd8542:C 2024-11-12T14:18:28,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:28,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421168201, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:28,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:28,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:28,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:28,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:28,233 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411121541bb0ec5194156aeec62d8505e65fb_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121541bb0ec5194156aeec62d8505e65fb_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:28,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/1ca0d415ed9f4c13a92b5e1948830b54, store: [table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:28,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/1ca0d415ed9f4c13a92b5e1948830b54 is 175, key is test_row_0/A:col10/1731421106900/Put/seqid=0 2024-11-12T14:18:28,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742234_1410 (size=31255) 2024-11-12T14:18:28,326 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:28,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421168319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:28,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:28,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:28,326 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:28,527 INFO [Thread-1469 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1469. 2024-11-12T14:18:28,527 DEBUG [Thread-1469 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:18:28,527 DEBUG [Thread-1469 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:28,527 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:18:28,527 DEBUG [Thread-1469 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:28,529 INFO [Thread-1471 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1471. 2024-11-12T14:18:28,529 DEBUG [Thread-1471 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:18:28,529 DEBUG [Thread-1471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:28,529 DEBUG [Thread-1471 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:28,529 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:18:28,530 INFO [Thread-1473 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1473. 2024-11-12T14:18:28,530 DEBUG [Thread-1473 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:18:28,530 DEBUG [Thread-1473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:28,530 DEBUG [Thread-1473 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:28,530 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:18:28,540 INFO [Thread-1467 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1467. 2024-11-12T14:18:28,540 DEBUG [Thread-1467 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:18:28,540 DEBUG [Thread-1467 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:28,540 DEBUG [Thread-1467 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:28,540 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:28,540 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:18:28,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421168539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:28,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:28,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:28,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:28,541 INFO [Thread-1465 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1465. 2024-11-12T14:18:28,541 DEBUG [Thread-1465 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:18:28,541 DEBUG [Thread-1465 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:28,541 DEBUG [Thread-1465 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:28,541 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:18:28,572 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/8326d065656d4df18176dcf1cf68babb as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/8326d065656d4df18176dcf1cf68babb 2024-11-12T14:18:28,577 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2f7e684f010a56c52cbae77227dd8542/A of 2f7e684f010a56c52cbae77227dd8542 into 8326d065656d4df18176dcf1cf68babb(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:28,577 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:28,577 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., storeName=2f7e684f010a56c52cbae77227dd8542/A, priority=13, startTime=1731421107736; duration=0sec 2024-11-12T14:18:28,577 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:28,577 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f7e684f010a56c52cbae77227dd8542:A 2024-11-12T14:18:28,638 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=306, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/1ca0d415ed9f4c13a92b5e1948830b54 2024-11-12T14:18:28,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/21684683b62346dd9a4d5f204625e6c1 is 50, key is test_row_0/B:col10/1731421106900/Put/seqid=0 2024-11-12T14:18:28,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742235_1411 (size=12301) 2024-11-12T14:18:28,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:28,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421168850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:28,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:28,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:28,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:28,921 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T14:18:29,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T14:18:29,048 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/21684683b62346dd9a4d5f204625e6c1 2024-11-12T14:18:29,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/ed359db3713f4f7b9028e32a3abc27b5 is 50, key is test_row_0/C:col10/1731421106900/Put/seqid=0 2024-11-12T14:18:29,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742236_1412 (size=12301) 2024-11-12T14:18:29,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:29,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:45712 deadline: 1731421169359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:29,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:29,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:18:29,465 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/ed359db3713f4f7b9028e32a3abc27b5 2024-11-12T14:18:29,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/1ca0d415ed9f4c13a92b5e1948830b54 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/1ca0d415ed9f4c13a92b5e1948830b54 2024-11-12T14:18:29,476 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/1ca0d415ed9f4c13a92b5e1948830b54, entries=150, sequenceid=306, filesize=30.5 K 2024-11-12T14:18:29,477 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/21684683b62346dd9a4d5f204625e6c1 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/21684683b62346dd9a4d5f204625e6c1 2024-11-12T14:18:29,480 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/21684683b62346dd9a4d5f204625e6c1, entries=150, sequenceid=306, filesize=12.0 K 2024-11-12T14:18:29,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/ed359db3713f4f7b9028e32a3abc27b5 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/ed359db3713f4f7b9028e32a3abc27b5 2024-11-12T14:18:29,484 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/ed359db3713f4f7b9028e32a3abc27b5, entries=150, sequenceid=306, filesize=12.0 K 2024-11-12T14:18:29,485 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 2f7e684f010a56c52cbae77227dd8542 in 1674ms, sequenceid=306, compaction requested=false 2024-11-12T14:18:29,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:29,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:29,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-12T14:18:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-12T14:18:29,488 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-12T14:18:29,488 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5950 sec 2024-11-12T14:18:29,489 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.6080 sec 2024-11-12T14:18:30,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:30,371 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 2f7e684f010a56c52cbae77227dd8542 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T14:18:30,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=A 2024-11-12T14:18:30,371 INFO [Thread-1456 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1456. 2024-11-12T14:18:30,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:30,371 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=B 2024-11-12T14:18:30,371 DEBUG [Thread-1456 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:18:30,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:30,372 DEBUG [Thread-1456 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:30,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=C 2024-11-12T14:18:30,372 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:30,372 DEBUG [Thread-1456 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:30,372 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:18:30,381 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112fed0db891a0d44e18e95df7808655e0c_2f7e684f010a56c52cbae77227dd8542 is 50, key is test_row_0/A:col10/1731421110370/Put/seqid=0 2024-11-12T14:18:30,384 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742237_1413 (size=12454) 2024-11-12T14:18:30,785 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:30,793 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112fed0db891a0d44e18e95df7808655e0c_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112fed0db891a0d44e18e95df7808655e0c_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:30,795 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/b06d2cbb1b71461db4e469a17c464a9a, store: [table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:30,796 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/b06d2cbb1b71461db4e469a17c464a9a is 175, key is test_row_0/A:col10/1731421110370/Put/seqid=0 2024-11-12T14:18:30,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742238_1414 (size=31255) 2024-11-12T14:18:31,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-12T14:18:31,050 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:18:31,202 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=328, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/b06d2cbb1b71461db4e469a17c464a9a 2024-11-12T14:18:31,213 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/e896199303a648959353841e2ac14bb6 is 50, key is test_row_0/B:col10/1731421110370/Put/seqid=0 2024-11-12T14:18:31,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742239_1415 (size=12301) 2024-11-12T14:18:31,620 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/e896199303a648959353841e2ac14bb6 2024-11-12T14:18:31,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/408c80ac26aa47e981bee93be09cfbd4 is 50, key is test_row_0/C:col10/1731421110370/Put/seqid=0 2024-11-12T14:18:31,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742240_1416 (size=12301) 2024-11-12T14:18:32,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=328 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/408c80ac26aa47e981bee93be09cfbd4 2024-11-12T14:18:32,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/b06d2cbb1b71461db4e469a17c464a9a as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/b06d2cbb1b71461db4e469a17c464a9a 2024-11-12T14:18:32,041 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/b06d2cbb1b71461db4e469a17c464a9a, entries=150, sequenceid=328, filesize=30.5 K 2024-11-12T14:18:32,042 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/e896199303a648959353841e2ac14bb6 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/e896199303a648959353841e2ac14bb6 2024-11-12T14:18:32,047 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/e896199303a648959353841e2ac14bb6, entries=150, sequenceid=328, filesize=12.0 K 2024-11-12T14:18:32,048 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/408c80ac26aa47e981bee93be09cfbd4 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/408c80ac26aa47e981bee93be09cfbd4 2024-11-12T14:18:32,053 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/408c80ac26aa47e981bee93be09cfbd4, entries=150, sequenceid=328, filesize=12.0 K 2024-11-12T14:18:32,054 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=0 B/0 for 2f7e684f010a56c52cbae77227dd8542 in 1683ms, sequenceid=328, compaction requested=true 2024-11-12T14:18:32,054 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:32,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f7e684f010a56c52cbae77227dd8542:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:18:32,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:32,054 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:32,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f7e684f010a56c52cbae77227dd8542:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:18:32,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:32,054 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:32,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 2f7e684f010a56c52cbae77227dd8542:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:18:32,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:32,055 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94515 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:32,055 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:32,055 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 2f7e684f010a56c52cbae77227dd8542/A is initiating minor compaction (all files) 2024-11-12T14:18:32,055 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 2f7e684f010a56c52cbae77227dd8542/B is initiating minor compaction (all files) 2024-11-12T14:18:32,055 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f7e684f010a56c52cbae77227dd8542/A in TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:32,055 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f7e684f010a56c52cbae77227dd8542/B in TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:32,055 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/8326d065656d4df18176dcf1cf68babb, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/1ca0d415ed9f4c13a92b5e1948830b54, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/b06d2cbb1b71461db4e469a17c464a9a] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp, totalSize=92.3 K 2024-11-12T14:18:32,055 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/bafd32d69416450f8afeadd9a7be0c18, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/21684683b62346dd9a4d5f204625e6c1, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/e896199303a648959353841e2ac14bb6] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp, totalSize=36.8 K 2024-11-12T14:18:32,055 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:32,055 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. files: [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/8326d065656d4df18176dcf1cf68babb, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/1ca0d415ed9f4c13a92b5e1948830b54, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/b06d2cbb1b71461db4e469a17c464a9a] 2024-11-12T14:18:32,056 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting bafd32d69416450f8afeadd9a7be0c18, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731421106442 2024-11-12T14:18:32,056 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8326d065656d4df18176dcf1cf68babb, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731421106442 2024-11-12T14:18:32,056 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 21684683b62346dd9a4d5f204625e6c1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731421106827 2024-11-12T14:18:32,056 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ca0d415ed9f4c13a92b5e1948830b54, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731421106827 2024-11-12T14:18:32,056 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting e896199303a648959353841e2ac14bb6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731421108186 2024-11-12T14:18:32,056 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b06d2cbb1b71461db4e469a17c464a9a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731421108186 2024-11-12T14:18:32,063 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f7e684f010a56c52cbae77227dd8542#B#compaction#342 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:32,064 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/27367a896c8542b5920787ac308fd10d is 50, key is test_row_0/B:col10/1731421110370/Put/seqid=0 2024-11-12T14:18:32,069 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:32,070 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742241_1417 (size=13153) 2024-11-12T14:18:32,074 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/27367a896c8542b5920787ac308fd10d as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/27367a896c8542b5920787ac308fd10d 2024-11-12T14:18:32,078 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2f7e684f010a56c52cbae77227dd8542/B of 2f7e684f010a56c52cbae77227dd8542 into 27367a896c8542b5920787ac308fd10d(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:32,078 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:32,078 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., storeName=2f7e684f010a56c52cbae77227dd8542/B, priority=13, startTime=1731421112054; duration=0sec 2024-11-12T14:18:32,079 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:32,079 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f7e684f010a56c52cbae77227dd8542:B 2024-11-12T14:18:32,079 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:32,079 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:32,079 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 2f7e684f010a56c52cbae77227dd8542/C is initiating minor compaction (all files) 2024-11-12T14:18:32,079 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 2f7e684f010a56c52cbae77227dd8542/C in TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:32,080 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/7e1e8b7836e34100b5a5f3afeff57277, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/ed359db3713f4f7b9028e32a3abc27b5, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/408c80ac26aa47e981bee93be09cfbd4] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp, totalSize=36.8 K 2024-11-12T14:18:32,080 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 7e1e8b7836e34100b5a5f3afeff57277, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=292, earliestPutTs=1731421106442 2024-11-12T14:18:32,080 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting ed359db3713f4f7b9028e32a3abc27b5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731421106827 2024-11-12T14:18:32,080 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 408c80ac26aa47e981bee93be09cfbd4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=328, earliestPutTs=1731421108186 2024-11-12T14:18:32,081 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112cde088927ae0473b89df8c00c62c4cbf_2f7e684f010a56c52cbae77227dd8542 store=[table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:32,095 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f7e684f010a56c52cbae77227dd8542#C#compaction#344 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:32,096 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/107eccdebf39421ca2b61b1aedef3a5e is 50, key is test_row_0/C:col10/1731421110370/Put/seqid=0 2024-11-12T14:18:32,104 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742242_1418 (size=13153) 2024-11-12T14:18:32,105 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112cde088927ae0473b89df8c00c62c4cbf_2f7e684f010a56c52cbae77227dd8542, store=[table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:32,105 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112cde088927ae0473b89df8c00c62c4cbf_2f7e684f010a56c52cbae77227dd8542 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:32,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742243_1419 (size=4469) 2024-11-12T14:18:32,510 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 2f7e684f010a56c52cbae77227dd8542#A#compaction#343 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:32,510 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/107eccdebf39421ca2b61b1aedef3a5e as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/107eccdebf39421ca2b61b1aedef3a5e 2024-11-12T14:18:32,510 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/7fdf9f050855423c84d44e34328f7372 is 175, key is test_row_0/A:col10/1731421110370/Put/seqid=0 2024-11-12T14:18:32,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742244_1420 (size=32107) 2024-11-12T14:18:32,514 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2f7e684f010a56c52cbae77227dd8542/C of 2f7e684f010a56c52cbae77227dd8542 into 107eccdebf39421ca2b61b1aedef3a5e(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:32,514 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:32,514 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., storeName=2f7e684f010a56c52cbae77227dd8542/C, priority=13, startTime=1731421112054; duration=0sec 2024-11-12T14:18:32,515 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:32,515 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f7e684f010a56c52cbae77227dd8542:C 2024-11-12T14:18:32,922 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/7fdf9f050855423c84d44e34328f7372 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/7fdf9f050855423c84d44e34328f7372 2024-11-12T14:18:32,928 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 2f7e684f010a56c52cbae77227dd8542/A of 2f7e684f010a56c52cbae77227dd8542 into 7fdf9f050855423c84d44e34328f7372(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:32,928 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 2f7e684f010a56c52cbae77227dd8542: 2024-11-12T14:18:32,928 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., storeName=2f7e684f010a56c52cbae77227dd8542/A, priority=13, startTime=1731421112054; duration=0sec 2024-11-12T14:18:32,928 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:32,928 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 2f7e684f010a56c52cbae77227dd8542:A 2024-11-12T14:18:34,929 WARN [regionserver/2b6d221c5cde:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 1 2024-11-12T14:18:34,931 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/f49dc62d10244c489e7fd7b93a9ac094, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/fffd1410429444babacfef9b4ecbd963, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/63d8ee2200a04fe09704bf83cc453ac2, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/253d58dff539446287d1cc5f7ec6d28c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/f5d6681bee3d490497683ea044ad14d7, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/7b0b79c81dae49a89d10a2b08be10374, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/3fb3491ecf5248ceb5038392f2c6856e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/20975ee9f2f14b04a6dd4a4d1ca89914, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/5cfa37444c0e4edaabd94ab45160e8c9, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/03a38b3b818548829dae79bce3c8828a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/c7b6e6eb9d8c421492cb290281d8efa7, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/b594768a7bd74f828a17e2ced94beb50, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/950bb442494f4ed38d8249012317c1b5, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/e3e4c53cfc1c408399672f86a0a8e948, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/62a601c754eb44749a63a83fb63784c3, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/2022d468f65e47b5accc8225f1d8c8f0, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/213ad4684708483face6f736113e3000, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/12b813b71f6649a59fac2f744d97c520, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/7b5d17242f7d4f1cb0f513ddf3770030, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/b183723aee7149dab7617fb7e5aabfdc, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/93c66b6db996477dbdfe3fed0a7864ca, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/8326d065656d4df18176dcf1cf68babb, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/f1eaea4991834e1d89e18a0ff59230aa, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/1ca0d415ed9f4c13a92b5e1948830b54, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/b06d2cbb1b71461db4e469a17c464a9a] to archive 2024-11-12T14:18:34,932 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T14:18:34,934 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/f49dc62d10244c489e7fd7b93a9ac094 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/f49dc62d10244c489e7fd7b93a9ac094 2024-11-12T14:18:34,936 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/fffd1410429444babacfef9b4ecbd963 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/fffd1410429444babacfef9b4ecbd963 2024-11-12T14:18:34,937 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/63d8ee2200a04fe09704bf83cc453ac2 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/63d8ee2200a04fe09704bf83cc453ac2 2024-11-12T14:18:34,939 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/253d58dff539446287d1cc5f7ec6d28c to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/253d58dff539446287d1cc5f7ec6d28c 2024-11-12T14:18:34,940 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/f5d6681bee3d490497683ea044ad14d7 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/f5d6681bee3d490497683ea044ad14d7 2024-11-12T14:18:34,941 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/7b0b79c81dae49a89d10a2b08be10374 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/7b0b79c81dae49a89d10a2b08be10374 2024-11-12T14:18:34,943 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/3fb3491ecf5248ceb5038392f2c6856e to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/3fb3491ecf5248ceb5038392f2c6856e 2024-11-12T14:18:34,944 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/20975ee9f2f14b04a6dd4a4d1ca89914 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/20975ee9f2f14b04a6dd4a4d1ca89914 2024-11-12T14:18:34,945 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/5cfa37444c0e4edaabd94ab45160e8c9 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/5cfa37444c0e4edaabd94ab45160e8c9 2024-11-12T14:18:34,946 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/03a38b3b818548829dae79bce3c8828a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/03a38b3b818548829dae79bce3c8828a 2024-11-12T14:18:34,947 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/c7b6e6eb9d8c421492cb290281d8efa7 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/c7b6e6eb9d8c421492cb290281d8efa7 2024-11-12T14:18:34,949 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/b594768a7bd74f828a17e2ced94beb50 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/b594768a7bd74f828a17e2ced94beb50 2024-11-12T14:18:34,950 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/950bb442494f4ed38d8249012317c1b5 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/950bb442494f4ed38d8249012317c1b5 2024-11-12T14:18:34,951 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/e3e4c53cfc1c408399672f86a0a8e948 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/e3e4c53cfc1c408399672f86a0a8e948 2024-11-12T14:18:34,953 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/62a601c754eb44749a63a83fb63784c3 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/62a601c754eb44749a63a83fb63784c3 2024-11-12T14:18:34,954 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/2022d468f65e47b5accc8225f1d8c8f0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/2022d468f65e47b5accc8225f1d8c8f0 2024-11-12T14:18:34,956 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/213ad4684708483face6f736113e3000 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/213ad4684708483face6f736113e3000 2024-11-12T14:18:34,958 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/12b813b71f6649a59fac2f744d97c520 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/12b813b71f6649a59fac2f744d97c520 2024-11-12T14:18:34,959 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/7b5d17242f7d4f1cb0f513ddf3770030 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/7b5d17242f7d4f1cb0f513ddf3770030 2024-11-12T14:18:34,960 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/b183723aee7149dab7617fb7e5aabfdc to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/b183723aee7149dab7617fb7e5aabfdc 2024-11-12T14:18:34,962 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/93c66b6db996477dbdfe3fed0a7864ca to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/93c66b6db996477dbdfe3fed0a7864ca 2024-11-12T14:18:34,965 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/8326d065656d4df18176dcf1cf68babb to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/8326d065656d4df18176dcf1cf68babb 2024-11-12T14:18:34,967 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/f1eaea4991834e1d89e18a0ff59230aa to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/f1eaea4991834e1d89e18a0ff59230aa 2024-11-12T14:18:34,968 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/1ca0d415ed9f4c13a92b5e1948830b54 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/1ca0d415ed9f4c13a92b5e1948830b54 2024-11-12T14:18:34,970 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/b06d2cbb1b71461db4e469a17c464a9a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/b06d2cbb1b71461db4e469a17c464a9a 2024-11-12T14:18:34,974 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/7f8c8f3c77ea45869164b9ae6908d27a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/b0ba951b5d2444cc8dbee6613760905d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/d95191cc7eab438e91d92ac3b40c4d73, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/5bd897e34f7741beae6731d109793d51, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/59496111405049abba90bcf128331ddf, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/ea818128bd2346ada1e47f2ece07945a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/776e78ef77864db89eacb08e8daacdb3, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/f647d15e5c3e4983868f7a9c3eb34a1a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/e2400819ef58409b9ed0b4a38c270c90, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/ba2f9df65c38482ba9dba7bae1d1bcf8, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/3e003a3f3f484a5fa47abdf95c7f0479, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/384c719e98d447e5867175ea75490a31, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/6c678fec983046cc8279199f8938e3b0, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/6d77d37f545d4674b3e8f98027c1d13f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/ac6d2777d6de472883b6743f6acbb055, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/9522709c3e4344b08885a1b6b00b621f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/225e10f8bd0249378bd7beb5c51ce9c4, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/80ff9e8bc72b4c4a8b0608956972e0d7, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/9b5c0a69156c4c548a5c7cf7129615d2, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/0d2aa89fda0e4facb8a1dbcd10813e3e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/b575908558044c6a9caf4b80c8c411c7, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/bafd32d69416450f8afeadd9a7be0c18, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/4e24c110e5884c1faafc7b3ed82c027d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/21684683b62346dd9a4d5f204625e6c1, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/e896199303a648959353841e2ac14bb6] to archive 2024-11-12T14:18:34,975 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T14:18:34,976 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/7f8c8f3c77ea45869164b9ae6908d27a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/7f8c8f3c77ea45869164b9ae6908d27a 2024-11-12T14:18:34,977 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/b0ba951b5d2444cc8dbee6613760905d to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/b0ba951b5d2444cc8dbee6613760905d 2024-11-12T14:18:34,977 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/d95191cc7eab438e91d92ac3b40c4d73 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/d95191cc7eab438e91d92ac3b40c4d73 2024-11-12T14:18:34,978 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/5bd897e34f7741beae6731d109793d51 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/5bd897e34f7741beae6731d109793d51 2024-11-12T14:18:34,979 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/59496111405049abba90bcf128331ddf to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/59496111405049abba90bcf128331ddf 2024-11-12T14:18:34,980 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/ea818128bd2346ada1e47f2ece07945a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/ea818128bd2346ada1e47f2ece07945a 2024-11-12T14:18:34,981 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/776e78ef77864db89eacb08e8daacdb3 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/776e78ef77864db89eacb08e8daacdb3 2024-11-12T14:18:34,981 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/f647d15e5c3e4983868f7a9c3eb34a1a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/f647d15e5c3e4983868f7a9c3eb34a1a 2024-11-12T14:18:34,982 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/e2400819ef58409b9ed0b4a38c270c90 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/e2400819ef58409b9ed0b4a38c270c90 2024-11-12T14:18:34,983 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/ba2f9df65c38482ba9dba7bae1d1bcf8 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/ba2f9df65c38482ba9dba7bae1d1bcf8 2024-11-12T14:18:34,984 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/3e003a3f3f484a5fa47abdf95c7f0479 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/3e003a3f3f484a5fa47abdf95c7f0479 2024-11-12T14:18:34,984 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/384c719e98d447e5867175ea75490a31 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/384c719e98d447e5867175ea75490a31 2024-11-12T14:18:34,985 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/6c678fec983046cc8279199f8938e3b0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/6c678fec983046cc8279199f8938e3b0 2024-11-12T14:18:34,986 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/6d77d37f545d4674b3e8f98027c1d13f to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/6d77d37f545d4674b3e8f98027c1d13f 2024-11-12T14:18:34,986 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/ac6d2777d6de472883b6743f6acbb055 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/ac6d2777d6de472883b6743f6acbb055 2024-11-12T14:18:34,987 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/9522709c3e4344b08885a1b6b00b621f to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/9522709c3e4344b08885a1b6b00b621f 2024-11-12T14:18:34,988 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/225e10f8bd0249378bd7beb5c51ce9c4 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/225e10f8bd0249378bd7beb5c51ce9c4 2024-11-12T14:18:34,989 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/80ff9e8bc72b4c4a8b0608956972e0d7 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/80ff9e8bc72b4c4a8b0608956972e0d7 2024-11-12T14:18:34,989 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/9b5c0a69156c4c548a5c7cf7129615d2 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/9b5c0a69156c4c548a5c7cf7129615d2 2024-11-12T14:18:34,990 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/0d2aa89fda0e4facb8a1dbcd10813e3e to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/0d2aa89fda0e4facb8a1dbcd10813e3e 2024-11-12T14:18:34,991 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/b575908558044c6a9caf4b80c8c411c7 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/b575908558044c6a9caf4b80c8c411c7 2024-11-12T14:18:34,991 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/bafd32d69416450f8afeadd9a7be0c18 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/bafd32d69416450f8afeadd9a7be0c18 2024-11-12T14:18:34,992 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/4e24c110e5884c1faafc7b3ed82c027d to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/4e24c110e5884c1faafc7b3ed82c027d 2024-11-12T14:18:34,993 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/21684683b62346dd9a4d5f204625e6c1 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/21684683b62346dd9a4d5f204625e6c1 2024-11-12T14:18:34,994 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/e896199303a648959353841e2ac14bb6 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/e896199303a648959353841e2ac14bb6 2024-11-12T14:18:34,996 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/1435205a24534b4ab4e893415251a1f9, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/d311f4ef4fa3480695a744a8d0d16a3c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/1199f00258d840e0aa069ec33588444f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/f674fee392e04efc9817bef9fcad98bc, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/476303ec4fab4d57b0f94b9424cd4705, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/d9db21c9851640f5bfa31a49b4d104af, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/e369bc5424ed4ef7bb5b4258b59e19f4, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/a2cbe591cb50452abea901598f5c5aab, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/c2624df9e93c4ca892f4488e17803644, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/23b53ab49cdb46fcab2cae7cd7b152aa, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/d271cfbd761b44198ec438af42ff7cff, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/171aa9e980134e08b6b08d5e10c30c3c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/8d12fd0a4b3b4e8993a8bf033fbe8573, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/895c59a422e446d8a556ad7fa45ae918, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/78a3c57fc5784a5692ef0f7129b42511, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/292c8f111d814b7ab00255e186ac3e84, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/b38f70076126487a99ebc0b9c8539156, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/a3f7a73ac2c9400cba844e52c0726634, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/0913550b6fd641a5a0bef601c737d28c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/12d9a88187d74186a74e87fa73148d86, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/53e63b1f765d4dcb8538e09443a664f4, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/7e1e8b7836e34100b5a5f3afeff57277, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/e6d7ea3104b846c0beb193063da09d82, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/ed359db3713f4f7b9028e32a3abc27b5, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/408c80ac26aa47e981bee93be09cfbd4] to archive 2024-11-12T14:18:34,997 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T14:18:34,998 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/1435205a24534b4ab4e893415251a1f9 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/1435205a24534b4ab4e893415251a1f9 2024-11-12T14:18:34,999 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/d311f4ef4fa3480695a744a8d0d16a3c to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/d311f4ef4fa3480695a744a8d0d16a3c 2024-11-12T14:18:35,000 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/1199f00258d840e0aa069ec33588444f to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/1199f00258d840e0aa069ec33588444f 2024-11-12T14:18:35,001 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/f674fee392e04efc9817bef9fcad98bc to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/f674fee392e04efc9817bef9fcad98bc 2024-11-12T14:18:35,001 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/476303ec4fab4d57b0f94b9424cd4705 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/476303ec4fab4d57b0f94b9424cd4705 2024-11-12T14:18:35,002 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/d9db21c9851640f5bfa31a49b4d104af to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/d9db21c9851640f5bfa31a49b4d104af 2024-11-12T14:18:35,003 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/e369bc5424ed4ef7bb5b4258b59e19f4 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/e369bc5424ed4ef7bb5b4258b59e19f4 2024-11-12T14:18:35,004 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/a2cbe591cb50452abea901598f5c5aab to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/a2cbe591cb50452abea901598f5c5aab 2024-11-12T14:18:35,005 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/c2624df9e93c4ca892f4488e17803644 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/c2624df9e93c4ca892f4488e17803644 2024-11-12T14:18:35,005 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/23b53ab49cdb46fcab2cae7cd7b152aa to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/23b53ab49cdb46fcab2cae7cd7b152aa 2024-11-12T14:18:35,006 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/d271cfbd761b44198ec438af42ff7cff to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/d271cfbd761b44198ec438af42ff7cff 2024-11-12T14:18:35,007 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/171aa9e980134e08b6b08d5e10c30c3c to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/171aa9e980134e08b6b08d5e10c30c3c 2024-11-12T14:18:35,008 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/8d12fd0a4b3b4e8993a8bf033fbe8573 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/8d12fd0a4b3b4e8993a8bf033fbe8573 2024-11-12T14:18:35,008 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/895c59a422e446d8a556ad7fa45ae918 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/895c59a422e446d8a556ad7fa45ae918 2024-11-12T14:18:35,009 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/78a3c57fc5784a5692ef0f7129b42511 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/78a3c57fc5784a5692ef0f7129b42511 2024-11-12T14:18:35,010 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/292c8f111d814b7ab00255e186ac3e84 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/292c8f111d814b7ab00255e186ac3e84 2024-11-12T14:18:35,011 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/b38f70076126487a99ebc0b9c8539156 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/b38f70076126487a99ebc0b9c8539156 2024-11-12T14:18:35,012 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/a3f7a73ac2c9400cba844e52c0726634 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/a3f7a73ac2c9400cba844e52c0726634 2024-11-12T14:18:35,012 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/0913550b6fd641a5a0bef601c737d28c to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/0913550b6fd641a5a0bef601c737d28c 2024-11-12T14:18:35,013 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/12d9a88187d74186a74e87fa73148d86 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/12d9a88187d74186a74e87fa73148d86 2024-11-12T14:18:35,014 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/53e63b1f765d4dcb8538e09443a664f4 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/53e63b1f765d4dcb8538e09443a664f4 2024-11-12T14:18:35,015 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/7e1e8b7836e34100b5a5f3afeff57277 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/7e1e8b7836e34100b5a5f3afeff57277 2024-11-12T14:18:35,016 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/e6d7ea3104b846c0beb193063da09d82 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/e6d7ea3104b846c0beb193063da09d82 2024-11-12T14:18:35,017 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/ed359db3713f4f7b9028e32a3abc27b5 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/ed359db3713f4f7b9028e32a3abc27b5 2024-11-12T14:18:35,018 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/2b6d221c5cde:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/408c80ac26aa47e981bee93be09cfbd4 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/408c80ac26aa47e981bee93be09cfbd4 2024-11-12T14:18:37,201 INFO [Thread-1462 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1462. 2024-11-12T14:18:37,201 DEBUG [Thread-1462 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:18:37,201 DEBUG [Thread-1462 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:37,201 DEBUG [Thread-1462 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:37,201 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:18:37,230 INFO [Thread-1454 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1454. 2024-11-12T14:18:37,230 DEBUG [Thread-1454 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:18:37,230 DEBUG [Thread-1454 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:37,230 DEBUG [Thread-1454 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:37,231 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:18:37,271 INFO [Thread-1460 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1460. 2024-11-12T14:18:37,271 DEBUG [Thread-1460 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:18:37,271 DEBUG [Thread-1460 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:37,271 DEBUG [Thread-1460 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:37,271 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:18:37,290 INFO [Thread-1458 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1458. 2024-11-12T14:18:37,290 DEBUG [Thread-1458 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:18:37,290 DEBUG [Thread-1458 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:37,290 DEBUG [Thread-1458 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:37,290 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:18:37,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T14:18:37,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-12T14:18:37,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-11-12T14:18:37,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-12T14:18:37,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-12T14:18:37,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 34 2024-11-12T14:18:37,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T14:18:37,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T14:18:37,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 931 2024-11-12T14:18:37,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2793 rows 2024-11-12T14:18:37,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 939 2024-11-12T14:18:37,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2815 rows 2024-11-12T14:18:37,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 950 2024-11-12T14:18:37,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2845 rows 2024-11-12T14:18:37,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 939 2024-11-12T14:18:37,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2814 rows 2024-11-12T14:18:37,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 943 2024-11-12T14:18:37,291 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2828 rows 2024-11-12T14:18:37,291 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T14:18:37,291 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T14:18:37,291 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:37,292 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:37,292 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:18:37,292 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-12T14:18:37,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T14:18:37,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T14:18:37,295 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731421117295"}]},"ts":"1731421117295"} 2024-11-12T14:18:37,296 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T14:18:37,296 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T14:18:37,297 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T14:18:37,299 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2f7e684f010a56c52cbae77227dd8542, UNASSIGN}] 2024-11-12T14:18:37,300 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2f7e684f010a56c52cbae77227dd8542, UNASSIGN 2024-11-12T14:18:37,301 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=103 updating hbase:meta row=2f7e684f010a56c52cbae77227dd8542, regionState=CLOSING, regionLocation=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:37,303 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=103, ppid=102, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=2f7e684f010a56c52cbae77227dd8542, UNASSIGN because future has completed 2024-11-12T14:18:37,303 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T14:18:37,303 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822}] 2024-11-12T14:18:37,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T14:18:37,456 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(122): Close 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:37,456 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T14:18:37,456 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1722): Closing 2f7e684f010a56c52cbae77227dd8542, disabling compactions & flushes 2024-11-12T14:18:37,456 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:37,456 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:37,456 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. after waiting 0 ms 2024-11-12T14:18:37,456 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:37,456 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(2902): Flushing 2f7e684f010a56c52cbae77227dd8542 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-12T14:18:37,456 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=A 2024-11-12T14:18:37,457 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:37,457 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=B 2024-11-12T14:18:37,457 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:37,457 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 2f7e684f010a56c52cbae77227dd8542, store=C 2024-11-12T14:18:37,457 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:37,462 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112cf18792d63874428b89ecf1f0029a709_2f7e684f010a56c52cbae77227dd8542 is 50, key is test_row_0/A:col10/1731421117289/Put/seqid=0 2024-11-12T14:18:37,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742245_1421 (size=12454) 2024-11-12T14:18:37,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T14:18:37,866 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:37,870 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112cf18792d63874428b89ecf1f0029a709_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112cf18792d63874428b89ecf1f0029a709_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:37,870 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/e71e68769db4461da989572f2373bc42, store: [table=TestAcidGuarantees family=A region=2f7e684f010a56c52cbae77227dd8542] 2024-11-12T14:18:37,871 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/e71e68769db4461da989572f2373bc42 is 175, key is test_row_0/A:col10/1731421117289/Put/seqid=0 2024-11-12T14:18:37,874 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742246_1422 (size=31255) 2024-11-12T14:18:37,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T14:18:38,275 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=338, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/e71e68769db4461da989572f2373bc42 2024-11-12T14:18:38,281 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/6ae03224074e4281b46792b5312eb7b4 is 50, key is test_row_0/B:col10/1731421117289/Put/seqid=0 2024-11-12T14:18:38,284 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742247_1423 (size=12301) 2024-11-12T14:18:38,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T14:18:38,684 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/6ae03224074e4281b46792b5312eb7b4 2024-11-12T14:18:38,690 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/0407b66e52df4a5f85e30f702ebd2603 is 50, key is test_row_0/C:col10/1731421117289/Put/seqid=0 2024-11-12T14:18:38,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742248_1424 (size=12301) 2024-11-12T14:18:39,093 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/0407b66e52df4a5f85e30f702ebd2603 2024-11-12T14:18:39,097 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/A/e71e68769db4461da989572f2373bc42 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/e71e68769db4461da989572f2373bc42 2024-11-12T14:18:39,100 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/e71e68769db4461da989572f2373bc42, entries=150, sequenceid=338, filesize=30.5 K 2024-11-12T14:18:39,100 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/B/6ae03224074e4281b46792b5312eb7b4 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/6ae03224074e4281b46792b5312eb7b4 2024-11-12T14:18:39,103 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/6ae03224074e4281b46792b5312eb7b4, entries=150, sequenceid=338, filesize=12.0 K 2024-11-12T14:18:39,104 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/.tmp/C/0407b66e52df4a5f85e30f702ebd2603 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/0407b66e52df4a5f85e30f702ebd2603 2024-11-12T14:18:39,107 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/0407b66e52df4a5f85e30f702ebd2603, entries=150, sequenceid=338, filesize=12.0 K 2024-11-12T14:18:39,107 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 2f7e684f010a56c52cbae77227dd8542 in 1651ms, sequenceid=338, compaction requested=false 2024-11-12T14:18:39,111 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/recovered.edits/341.seqid, newMaxSeqId=341, maxSeqId=4 2024-11-12T14:18:39,111 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. 2024-11-12T14:18:39,111 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] regionserver.HRegion(1676): Region close journal for 2f7e684f010a56c52cbae77227dd8542: Waiting for close lock at 1731421117456Running coprocessor pre-close hooks at 1731421117456Disabling compacts and flushes for region at 1731421117456Disabling writes for close at 1731421117456Obtaining lock to block concurrent updates at 1731421117456Preparing flush snapshotting stores in 2f7e684f010a56c52cbae77227dd8542 at 1731421117456Finished memstore snapshotting TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731421117457 (+1 ms)Flushing stores of TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542. at 1731421117457Flushing 2f7e684f010a56c52cbae77227dd8542/A: creating writer at 1731421117457Flushing mob file 2f7e684f010a56c52cbae77227dd8542/A: appending metadata at 1731421117462 (+5 ms)Flushing mob file 2f7e684f010a56c52cbae77227dd8542/A: closing flushed file at 1731421117462Flushing 2f7e684f010a56c52cbae77227dd8542/A: appending metadata at 1731421117870 (+408 ms)Flushing 2f7e684f010a56c52cbae77227dd8542/A: closing flushed file at 1731421117871 (+1 ms)Flushing 2f7e684f010a56c52cbae77227dd8542/B: creating writer at 1731421118278 (+407 ms)Flushing 2f7e684f010a56c52cbae77227dd8542/B: appending metadata at 1731421118280 (+2 ms)Flushing 2f7e684f010a56c52cbae77227dd8542/B: closing flushed file at 1731421118280Flushing 2f7e684f010a56c52cbae77227dd8542/C: creating writer at 1731421118687 (+407 ms)Flushing 2f7e684f010a56c52cbae77227dd8542/C: appending metadata at 1731421118689 (+2 ms)Flushing 2f7e684f010a56c52cbae77227dd8542/C: closing flushed file at 1731421118689Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7af67407: reopening flushed file at 1731421119096 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6bddbd79: reopening flushed file at 1731421119100 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@68507ffd: reopening flushed file at 1731421119103 (+3 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for 2f7e684f010a56c52cbae77227dd8542 in 1651ms, sequenceid=338, compaction requested=false at 1731421119107 (+4 ms)Writing region close event to WAL at 1731421119109 (+2 ms)Running coprocessor post-close hooks at 1731421119111 (+2 ms)Closed at 1731421119111 2024-11-12T14:18:39,113 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=104}] handler.UnassignRegionHandler(157): Closed 2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:39,113 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=103 updating hbase:meta row=2f7e684f010a56c52cbae77227dd8542, regionState=CLOSED 2024-11-12T14:18:39,114 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=104, ppid=103, state=RUNNABLE, hasLock=false; CloseRegionProcedure 2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 because future has completed 2024-11-12T14:18:39,116 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-12T14:18:39,116 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; CloseRegionProcedure 2f7e684f010a56c52cbae77227dd8542, server=2b6d221c5cde,42925,1731420990822 in 1.8120 sec 2024-11-12T14:18:39,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=103, resume processing ppid=102 2024-11-12T14:18:39,118 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, ppid=102, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=2f7e684f010a56c52cbae77227dd8542, UNASSIGN in 1.8180 sec 2024-11-12T14:18:39,119 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-12T14:18:39,119 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8210 sec 2024-11-12T14:18:39,120 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731421119120"}]},"ts":"1731421119120"} 2024-11-12T14:18:39,122 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T14:18:39,122 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T14:18:39,123 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8300 sec 2024-11-12T14:18:39,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-12T14:18:39,439 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:18:39,439 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-12T14:18:39,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:18:39,441 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=105, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:18:39,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T14:18:39,441 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=105, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:18:39,443 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:39,446 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A, FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B, FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C, FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/recovered.edits] 2024-11-12T14:18:39,448 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/7fdf9f050855423c84d44e34328f7372 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/7fdf9f050855423c84d44e34328f7372 2024-11-12T14:18:39,449 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/e71e68769db4461da989572f2373bc42 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/A/e71e68769db4461da989572f2373bc42 2024-11-12T14:18:39,451 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/27367a896c8542b5920787ac308fd10d to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/27367a896c8542b5920787ac308fd10d 2024-11-12T14:18:39,451 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/6ae03224074e4281b46792b5312eb7b4 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/B/6ae03224074e4281b46792b5312eb7b4 2024-11-12T14:18:39,453 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/0407b66e52df4a5f85e30f702ebd2603 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/0407b66e52df4a5f85e30f702ebd2603 2024-11-12T14:18:39,454 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/107eccdebf39421ca2b61b1aedef3a5e to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/C/107eccdebf39421ca2b61b1aedef3a5e 2024-11-12T14:18:39,456 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/recovered.edits/341.seqid to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542/recovered.edits/341.seqid 2024-11-12T14:18:39,456 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:39,456 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T14:18:39,456 DEBUG [PEWorker-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T14:18:39,457 DEBUG [PEWorker-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-12T14:18:39,459 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411120856ddd6a821422e8cb3235d49aa94f1_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411120856ddd6a821422e8cb3235d49aa94f1_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:39,460 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411120ce8703550374098b120e582dcf84f24_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411120ce8703550374098b120e582dcf84f24_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:39,462 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121541bb0ec5194156aeec62d8505e65fb_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121541bb0ec5194156aeec62d8505e65fb_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:39,462 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121b5ec5f7e58640ed941710fd1d259a88_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411121b5ec5f7e58640ed941710fd1d259a88_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:39,463 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122024926627e54f6fbb9dd1333eb30931_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411122024926627e54f6fbb9dd1333eb30931_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:39,464 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111222dc12a3d6cc4a028811c0280a78b653_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111222dc12a3d6cc4a028811c0280a78b653_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:39,465 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111229dff9c43c3c4853b1b9f482ad193a91_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111229dff9c43c3c4853b1b9f482ad193a91_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:39,466 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111236113c092ec94eefa36353a88e2dd66c_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111236113c092ec94eefa36353a88e2dd66c_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:39,467 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124e90c9dc159a4d32a6aaa16982560372_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124e90c9dc159a4d32a6aaa16982560372_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:39,468 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126d266f617c84460592cffa2022fb1575_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411126d266f617c84460592cffa2022fb1575_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:39,469 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111272211b325be14074a5e9488adce53e40_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111272211b325be14074a5e9488adce53e40_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:39,469 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112885af978205e46519d20699715257653_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112885af978205e46519d20699715257653_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:39,470 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411128d822346371c4aa3ad409ffed592b254_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411128d822346371c4aa3ad409ffed592b254_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:39,471 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129651440473d94e47909daf8586151122_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129651440473d94e47909daf8586151122_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:39,472 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111297b6a474e61c4c3aba04820dc70ad1f9_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111297b6a474e61c4c3aba04820dc70ad1f9_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:39,473 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112cf18792d63874428b89ecf1f0029a709_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112cf18792d63874428b89ecf1f0029a709_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:39,475 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ed8a597c7eeb4ebb80c8947a1569e9f5_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ed8a597c7eeb4ebb80c8947a1569e9f5_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:39,476 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f0890af18a2b43d9bcbb70a956dafa13_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f0890af18a2b43d9bcbb70a956dafa13_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:39,478 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f6d2b4160a484d238b350b23238c2082_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112f6d2b4160a484d238b350b23238c2082_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:39,480 DEBUG [PEWorker-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112fed0db891a0d44e18e95df7808655e0c_2f7e684f010a56c52cbae77227dd8542 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112fed0db891a0d44e18e95df7808655e0c_2f7e684f010a56c52cbae77227dd8542 2024-11-12T14:18:39,480 DEBUG [PEWorker-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T14:18:39,483 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=105, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:18:39,485 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T14:18:39,487 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T14:18:39,487 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=105, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:18:39,488 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T14:18:39,488 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731421119488"}]},"ts":"9223372036854775807"} 2024-11-12T14:18:39,490 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T14:18:39,490 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 2f7e684f010a56c52cbae77227dd8542, NAME => 'TestAcidGuarantees,,1731421085408.2f7e684f010a56c52cbae77227dd8542.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T14:18:39,490 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T14:18:39,490 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731421119490"}]},"ts":"9223372036854775807"} 2024-11-12T14:18:39,491 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T14:18:39,492 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=105, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:18:39,493 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 53 msec 2024-11-12T14:18:39,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-12T14:18:39,549 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T14:18:39,549 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:18:39,556 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobScanAtomicity Thread=248 (was 250), OpenFileDescriptor=451 (was 453), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=858 (was 1140), ProcessCount=11 (was 11), AvailableMemoryMB=1334 (was 2561) 2024-11-12T14:18:39,565 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=248, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=858, ProcessCount=11, AvailableMemoryMB=1334 2024-11-12T14:18:39,566 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T14:18:39,566 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T14:18:39,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=106, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T14:18:39,568 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T14:18:39,568 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:39,568 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 106 2024-11-12T14:18:39,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-12T14:18:39,568 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T14:18:39,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742249_1425 (size=1061) 2024-11-12T14:18:39,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-12T14:18:39,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-12T14:18:39,974 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3 2024-11-12T14:18:39,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742250_1426 (size=53) 2024-11-12T14:18:40,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-12T14:18:40,335 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-12T14:18:40,380 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:18:40,380 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 33074d5f6b8c90fb791f1838297603c6, disabling compactions & flushes 2024-11-12T14:18:40,380 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:40,380 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:40,381 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. after waiting 1 ms 2024-11-12T14:18:40,381 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:40,381 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:40,381 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 33074d5f6b8c90fb791f1838297603c6: Waiting for close lock at 1731421120380Disabling compacts and flushes for region at 1731421120380Disabling writes for close at 1731421120381 (+1 ms)Writing region close event to WAL at 1731421120381Closed at 1731421120381 2024-11-12T14:18:40,382 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T14:18:40,382 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731421120382"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731421120382"}]},"ts":"1731421120382"} 2024-11-12T14:18:40,383 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T14:18:40,383 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T14:18:40,383 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731421120383"}]},"ts":"1731421120383"} 2024-11-12T14:18:40,384 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T14:18:40,385 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=33074d5f6b8c90fb791f1838297603c6, ASSIGN}] 2024-11-12T14:18:40,385 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=33074d5f6b8c90fb791f1838297603c6, ASSIGN 2024-11-12T14:18:40,386 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=33074d5f6b8c90fb791f1838297603c6, ASSIGN; state=OFFLINE, location=2b6d221c5cde,42925,1731420990822; forceNewPlan=false, retain=false 2024-11-12T14:18:40,537 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=33074d5f6b8c90fb791f1838297603c6, regionState=OPENING, regionLocation=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:40,538 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=107, ppid=106, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=33074d5f6b8c90fb791f1838297603c6, ASSIGN because future has completed 2024-11-12T14:18:40,539 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; OpenRegionProcedure 33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822}] 2024-11-12T14:18:40,695 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:40,695 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7752): Opening region: {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:18:40,695 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:40,695 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:18:40,695 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7794): checking encryption for 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:40,695 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(7797): checking classloading for 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:40,696 INFO [StoreOpener-33074d5f6b8c90fb791f1838297603c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:40,697 INFO [StoreOpener-33074d5f6b8c90fb791f1838297603c6-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T14:18:40,697 INFO [StoreOpener-33074d5f6b8c90fb791f1838297603c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33074d5f6b8c90fb791f1838297603c6 columnFamilyName A 2024-11-12T14:18:40,697 DEBUG [StoreOpener-33074d5f6b8c90fb791f1838297603c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:40,698 INFO [StoreOpener-33074d5f6b8c90fb791f1838297603c6-1 {}] regionserver.HStore(327): Store=33074d5f6b8c90fb791f1838297603c6/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:18:40,698 INFO [StoreOpener-33074d5f6b8c90fb791f1838297603c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:40,698 INFO [StoreOpener-33074d5f6b8c90fb791f1838297603c6-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T14:18:40,699 INFO [StoreOpener-33074d5f6b8c90fb791f1838297603c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33074d5f6b8c90fb791f1838297603c6 columnFamilyName B 2024-11-12T14:18:40,699 DEBUG [StoreOpener-33074d5f6b8c90fb791f1838297603c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:40,699 INFO [StoreOpener-33074d5f6b8c90fb791f1838297603c6-1 {}] regionserver.HStore(327): Store=33074d5f6b8c90fb791f1838297603c6/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:18:40,699 INFO [StoreOpener-33074d5f6b8c90fb791f1838297603c6-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:40,700 INFO [StoreOpener-33074d5f6b8c90fb791f1838297603c6-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T14:18:40,700 INFO [StoreOpener-33074d5f6b8c90fb791f1838297603c6-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 33074d5f6b8c90fb791f1838297603c6 columnFamilyName C 2024-11-12T14:18:40,700 DEBUG [StoreOpener-33074d5f6b8c90fb791f1838297603c6-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:18:40,700 INFO [StoreOpener-33074d5f6b8c90fb791f1838297603c6-1 {}] regionserver.HStore(327): Store=33074d5f6b8c90fb791f1838297603c6/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:18:40,701 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:40,701 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1038): replaying wal for 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:40,701 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:40,702 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:40,702 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1048): stopping wal replay for 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:40,702 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1060): Cleaning up temporary data for 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:40,703 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T14:18:40,704 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1093): writing seq id for 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:40,705 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:18:40,706 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1114): Opened 33074d5f6b8c90fb791f1838297603c6; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66894911, jitterRate=-0.0031881481409072876}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T14:18:40,706 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:40,707 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegion(1006): Region open journal for 33074d5f6b8c90fb791f1838297603c6: Running coprocessor pre-open hook at 1731421120695Writing region info on filesystem at 1731421120695Initializing all the Stores at 1731421120696 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731421120696Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731421120696Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731421120696Cleaning up temporary data from old regions at 1731421120702 (+6 ms)Running coprocessor post-open hooks at 1731421120706 (+4 ms)Region opened successfully at 1731421120706 2024-11-12T14:18:40,707 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., pid=108, masterSystemTime=1731421120692 2024-11-12T14:18:40,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-12T14:18:40,709 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:40,709 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=108}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:40,709 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=107 updating hbase:meta row=33074d5f6b8c90fb791f1838297603c6, regionState=OPEN, openSeqNum=2, regionLocation=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:40,710 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=108, ppid=107, state=RUNNABLE, hasLock=false; OpenRegionProcedure 33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 because future has completed 2024-11-12T14:18:40,710 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45591 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=2b6d221c5cde,42925,1731420990822, table=TestAcidGuarantees, region=33074d5f6b8c90fb791f1838297603c6. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T14:18:40,713 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-12T14:18:40,713 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; OpenRegionProcedure 33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 in 172 msec 2024-11-12T14:18:40,714 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=107, resume processing ppid=106 2024-11-12T14:18:40,714 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, ppid=106, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=33074d5f6b8c90fb791f1838297603c6, ASSIGN in 328 msec 2024-11-12T14:18:40,715 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T14:18:40,715 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731421120715"}]},"ts":"1731421120715"} 2024-11-12T14:18:40,716 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T14:18:40,716 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=106, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T14:18:40,717 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1510 sec 2024-11-12T14:18:41,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=106 2024-11-12T14:18:41,729 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:18:41,730 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fe8d753, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,730 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:18:41,731 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:18:41,732 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:18:41,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:18:41,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:18:41,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b2637a4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:18:41,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:18:41,732 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:41,733 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53338, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:18:41,734 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d2bab3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,734 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:18:41,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:18:41,735 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:18:41,736 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38610, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:18:41,736 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T14:18:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 2b6d221c5cde,45591,1731420989939 2024-11-12T14:18:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@3209255a 2024-11-12T14:18:41,737 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T14:18:41,738 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53348, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T14:18:41,739 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dee51f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,739 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:18:41,739 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:18:41,740 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:18:41,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:18:41,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:18:41,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@629249fb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:18:41,740 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:18:41,741 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:41,741 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53356, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:18:41,742 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4885664c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,743 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@d395438, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,743 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:18:41,743 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:18:41,743 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:18:41,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:18:41,743 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:18:41,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2a388380, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:18:41,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:18:41,744 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:41,745 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53376, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:18:41,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d4e459c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,746 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b2d6985, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,746 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:18:41,746 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:18:41,746 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:18:41,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:18:41,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:18:41,746 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a24463b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:18:41,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:18:41,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:41,747 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53384, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:18:41,748 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1be05eca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,748 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@490627bb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,748 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:18:41,748 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:18:41,749 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:18:41,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:18:41,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:18:41,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24fa49c8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:18:41,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:18:41,749 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:41,750 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53414, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:18:41,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@35d26f45, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,752 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e16e9b9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,752 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:18:41,752 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:18:41,753 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:18:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:18:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:18:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1d808f79, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:18:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:18:41,753 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:41,754 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53430, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:18:41,754 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53cfc5ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,756 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@794001fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,756 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:18:41,756 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:18:41,757 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:18:41,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:18:41,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:18:41,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7391de98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:18:41,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:18:41,757 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:41,758 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53456, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:18:41,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@613b0cb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,759 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f3c4fbc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,759 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:18:41,759 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:18:41,759 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:18:41,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:18:41,759 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:18:41,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54eab094, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:18:41,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:18:41,760 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:41,760 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53470, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:18:41,761 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bde3bca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,761 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@732b24ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,761 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:18:41,761 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:18:41,762 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:18:41,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:18:41,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:18:41,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@28c10c09, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:18:41,762 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:18:41,763 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:41,763 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53496, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:18:41,764 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@612d734, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,765 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eaa8538, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,765 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:18:41,765 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:18:41,766 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:18:41,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:18:41,766 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:18:41,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@798f657a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:18:41,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:18:41,767 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:41,768 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53516, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:18:41,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c75d4e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,769 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67669ab8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,769 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:18:41,769 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:18:41,770 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:18:41,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:18:41,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:18:41,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a5bb250, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,770 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:18:41,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:18:41,771 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:18:41,771 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:53528, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:18:41,772 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41371c1a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:18:41,772 DEBUG [Thread-1928 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:18:41,772 DEBUG [Thread-1941 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:18:41,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:18:41,773 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:18:41,773 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:18:41,773 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:18:41,774 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38624, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:18:41,774 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38622, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:18:41,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2] 2024-11-12T14:18:41,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2] 2024-11-12T14:18:41,776 DEBUG [Thread-1932 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:18:41,777 DEBUG [Thread-1943 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:18:41,777 DEBUG [Thread-1937 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:18:41,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:18:41,777 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:18:41,778 DEBUG [Thread-1934 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:18:41,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:18:41,778 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:18:41,779 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:18:41,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:18:41,779 DEBUG [Thread-1934 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:18:41,779 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38640, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:18:41,779 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:18:41,779 DEBUG [Thread-1934 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:18:41,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-12T14:18:41,780 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38654, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:18:41,780 DEBUG [Thread-1930 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:18:41,780 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38656, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:18:41,781 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38650, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:18:41,781 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2] 2024-11-12T14:18:41,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T14:18:41,781 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:18:41,781 DEBUG [Thread-1926 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:18:41,781 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:18:41,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:18:41,782 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:18:41,782 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:18:41,783 DEBUG [Thread-1926 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:18:41,783 DEBUG [Thread-1926 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:18:41,784 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38658, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:18:41,784 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38668, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:18:41,785 DEBUG [Thread-1939 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:18:41,785 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2] 2024-11-12T14:18:41,785 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:18:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:18:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2] 2024-11-12T14:18:41,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2] 2024-11-12T14:18:41,787 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38680, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:18:41,787 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2] 2024-11-12T14:18:41,787 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2] 2024-11-12T14:18:41,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2] 2024-11-12T14:18:41,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:41,792 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T14:18:41,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:18:41,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:41,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:18:41,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:41,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:18:41,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:41,804 DEBUG [Thread-1945 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:18:41,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:18:41,805 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:18:41,806 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:18:41,808 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2] 2024-11-12T14:18:41,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731421121811, max timestamp=1731421121821Num uniques 859; 2024-11-12T14:18:41,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T14:18:41,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731421121811, max timestamp=1731421121821Num uniques 859; 2024-11-12T14:18:41,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731421121811, max timestamp=1731421121821Num uniques 859; 2024-11-12T14:18:41,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T14:18:41,824 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T14:18:41,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/3c021a86f0fa48c7aead303b5250e346 is 50, key is test_row_0/A:col10/1731421121792/Put/seqid=0 2024-11-12T14:18:41,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742251_1427 (size=12001) 2024-11-12T14:18:41,879 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/3c021a86f0fa48c7aead303b5250e346 2024-11-12T14:18:41,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:41,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421181862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:41,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:41,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:41,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:41,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:41,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421181863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:41,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:41,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:41,882 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:41,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:41,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38622 deadline: 1731421181864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:41,886 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:41,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:41,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1731421181864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:41,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:41,886 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:41,886 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:41,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:41,887 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:41,887 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:41,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421181865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:41,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:41,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:41,887 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:41,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T14:18:41,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/0b2a1dd58bdb41b6a5df074e53996cea is 50, key is test_row_0/B:col10/1731421121792/Put/seqid=0 2024-11-12T14:18:41,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T14:18:41,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:41,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:41,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:41,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:41,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:41,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:41,965 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742252_1428 (size=12001) 2024-11-12T14:18:41,966 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/0b2a1dd58bdb41b6a5df074e53996cea 2024-11-12T14:18:41,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:41,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421181989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:41,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:41,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1731421181990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:41,998 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:41,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:41,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:41,999 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:41,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421181990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:41,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:41,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:41,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:41,999 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:41,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421181990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:41,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:41,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:41,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38622 deadline: 1731421181990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:41,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:41,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:42,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:42,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:42,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/f72f91d0aba34d66bef60673686bb892 is 50, key is test_row_0/C:col10/1731421121792/Put/seqid=0 2024-11-12T14:18:42,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742253_1429 (size=12001) 2024-11-12T14:18:42,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T14:18:42,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:42,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:42,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:42,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:42,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:42,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:42,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T14:18:42,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:42,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1731421182209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:42,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:42,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421182209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:42,218 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:42,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:42,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421182209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:42,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:42,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,219 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:42,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421182209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:42,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,219 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:42,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:42,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38622 deadline: 1731421182209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:42,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,223 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:42,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,224 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:42,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T14:18:42,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:42,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:42,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:42,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:42,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:42,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:42,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T14:18:42,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:42,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:42,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:42,402 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:42,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:42,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:42,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T14:18:42,414 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/f72f91d0aba34d66bef60673686bb892 2024-11-12T14:18:42,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/3c021a86f0fa48c7aead303b5250e346 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/3c021a86f0fa48c7aead303b5250e346 2024-11-12T14:18:42,424 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/3c021a86f0fa48c7aead303b5250e346, entries=150, sequenceid=12, filesize=11.7 K 2024-11-12T14:18:42,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/0b2a1dd58bdb41b6a5df074e53996cea as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/0b2a1dd58bdb41b6a5df074e53996cea 2024-11-12T14:18:42,428 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/0b2a1dd58bdb41b6a5df074e53996cea, entries=150, sequenceid=12, filesize=11.7 K 2024-11-12T14:18:42,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/f72f91d0aba34d66bef60673686bb892 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/f72f91d0aba34d66bef60673686bb892 2024-11-12T14:18:42,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/f72f91d0aba34d66bef60673686bb892, entries=150, sequenceid=12, filesize=11.7 K 2024-11-12T14:18:42,435 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 33074d5f6b8c90fb791f1838297603c6 in 644ms, sequenceid=12, compaction requested=false 2024-11-12T14:18:42,435 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:42,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:42,534 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-12T14:18:42,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:18:42,534 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:18:42,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:18:42,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:18:42,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:18:42,535 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:18:42,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/289bb14d29db4984a404908620d4f3b2 is 50, key is test_row_0/A:col10/1731421121864/Put/seqid=0 2024-11-12T14:18:42,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T14:18:42,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:42,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:42,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:42,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:42,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:42,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:42,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742254_1430 (size=14341) 2024-11-12T14:18:42,567 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/289bb14d29db4984a404908620d4f3b2 2024-11-12T14:18:42,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:42,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421182544, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:42,569 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:42,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1731421182547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,569 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,570 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:42,570 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:42,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421182547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:42,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,570 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:42,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,570 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:42,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/c7f6d777c75f4205812f0ab40bf47312 is 50, key is test_row_0/B:col10/1731421121864/Put/seqid=0 2024-11-12T14:18:42,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38622 deadline: 1731421182568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:42,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:42,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421182570, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:42,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,588 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:42,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,588 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:42,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742255_1431 (size=12001) 2024-11-12T14:18:42,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:42,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1731421182679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:42,693 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:42,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421182680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:42,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,693 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:42,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,693 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:42,694 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:42,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421182681, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:42,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,694 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:42,702 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:42,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38622 deadline: 1731421182689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:42,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,703 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:42,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T14:18:42,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:42,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:42,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:42,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:42,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:42,712 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:42,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421182699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:42,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:42,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,713 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:42,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T14:18:42,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:42,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:42,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:42,864 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:42,864 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:42,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:42,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:42,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421182899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:42,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:42,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1731421182899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:42,906 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:42,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:42,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:42,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421182899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:42,906 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,907 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:42,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T14:18:42,919 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:42,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38622 deadline: 1731421182908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:42,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,920 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:42,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:42,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421182919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:42,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:42,930 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:42,990 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/c7f6d777c75f4205812f0ab40bf47312 2024-11-12T14:18:42,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/2683857647134d86b00263ec9c80dc44 is 50, key is test_row_0/C:col10/1731421121864/Put/seqid=0 2024-11-12T14:18:43,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742256_1432 (size=12001) 2024-11-12T14:18:43,016 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T14:18:43,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:43,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:43,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:43,016 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:43,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:43,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:43,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T14:18:43,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:43,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:43,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:43,168 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:43,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:43,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:43,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:43,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421183209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:43,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:43,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:43,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:43,219 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:43,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421183209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:43,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:43,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:43,219 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:43,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:43,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1731421183226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:43,236 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:43,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:43,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:43,236 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:43,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38622 deadline: 1731421183231, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:43,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:43,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:43,237 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:43,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:43,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421183239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:43,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:43,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:43,254 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:43,320 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T14:18:43,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:43,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:43,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:43,321 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:43,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:43,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:43,406 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/2683857647134d86b00263ec9c80dc44 2024-11-12T14:18:43,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/289bb14d29db4984a404908620d4f3b2 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/289bb14d29db4984a404908620d4f3b2 2024-11-12T14:18:43,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/289bb14d29db4984a404908620d4f3b2, entries=200, sequenceid=48, filesize=14.0 K 2024-11-12T14:18:43,419 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/c7f6d777c75f4205812f0ab40bf47312 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/c7f6d777c75f4205812f0ab40bf47312 2024-11-12T14:18:43,425 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/c7f6d777c75f4205812f0ab40bf47312, entries=150, sequenceid=48, filesize=11.7 K 2024-11-12T14:18:43,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/2683857647134d86b00263ec9c80dc44 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/2683857647134d86b00263ec9c80dc44 2024-11-12T14:18:43,430 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/2683857647134d86b00263ec9c80dc44, entries=150, sequenceid=48, filesize=11.7 K 2024-11-12T14:18:43,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=53.67 KB/54960 for 33074d5f6b8c90fb791f1838297603c6 in 896ms, sequenceid=48, compaction requested=false 2024-11-12T14:18:43,431 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:43,473 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-12T14:18:43,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:43,474 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T14:18:43,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:18:43,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:43,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:18:43,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:43,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:18:43,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:43,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/95b1d08c39a54e90982b67926820508b is 50, key is test_row_0/A:col10/1731421122543/Put/seqid=0 2024-11-12T14:18:43,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742257_1433 (size=12001) 2024-11-12T14:18:43,498 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/95b1d08c39a54e90982b67926820508b 2024-11-12T14:18:43,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/8406dac22c064be6a25f8fba349372c9 is 50, key is test_row_0/B:col10/1731421122543/Put/seqid=0 2024-11-12T14:18:43,549 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742258_1434 (size=12001) 2024-11-12T14:18:43,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:43,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:43,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421183782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:43,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:43,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421183784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:43,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:43,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:43,798 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:43,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:43,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:43,798 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:43,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731421123776, max timestamp=1731421123793Num uniques 859; 2024-11-12T14:18:43,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T14:18:43,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731421123776, max timestamp=1731421123795Num uniques 859; 2024-11-12T14:18:43,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T14:18:43,799 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731421123776, max timestamp=1731421123793Num uniques 859; 2024-11-12T14:18:43,800 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T14:18:43,855 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:43,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421183848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:43,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:43,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:43,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:43,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:43,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38622 deadline: 1731421183849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:43,866 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:43,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:43,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:43,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1731421183855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:43,866 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:43,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:43,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:43,867 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:43,916 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:43,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421183909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:43,917 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:43,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:43,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421183910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:43,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:43,917 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:43,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:43,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:43,917 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T14:18:43,949 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/8406dac22c064be6a25f8fba349372c9 2024-11-12T14:18:43,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/e3099985d2ae40b59b6a1890af491060 is 50, key is test_row_0/C:col10/1731421122543/Put/seqid=0 2024-11-12T14:18:43,959 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742259_1435 (size=12001) 2024-11-12T14:18:43,964 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:43,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421183959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:43,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:43,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:43,965 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:43,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:43,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38622 deadline: 1731421183969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:43,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:43,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1731421183969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:43,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:43,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:43,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:43,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:43,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:43,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:44,124 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:44,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421184119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:44,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:44,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:44,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:44,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:44,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421184130, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:44,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:44,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:44,144 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:44,176 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:44,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421184169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:44,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:44,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:44,177 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:44,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38622 deadline: 1731421184189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:44,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:44,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:44,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1731421184190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:44,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:44,195 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:44,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:44,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:44,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:44,360 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=59 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/e3099985d2ae40b59b6a1890af491060 2024-11-12T14:18:44,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/95b1d08c39a54e90982b67926820508b as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/95b1d08c39a54e90982b67926820508b 2024-11-12T14:18:44,367 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/95b1d08c39a54e90982b67926820508b, entries=150, sequenceid=59, filesize=11.7 K 2024-11-12T14:18:44,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/8406dac22c064be6a25f8fba349372c9 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/8406dac22c064be6a25f8fba349372c9 2024-11-12T14:18:44,374 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/8406dac22c064be6a25f8fba349372c9, entries=150, sequenceid=59, filesize=11.7 K 2024-11-12T14:18:44,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/e3099985d2ae40b59b6a1890af491060 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/e3099985d2ae40b59b6a1890af491060 2024-11-12T14:18:44,378 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/e3099985d2ae40b59b6a1890af491060, entries=150, sequenceid=59, filesize=11.7 K 2024-11-12T14:18:44,378 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=127.47 KB/130530 for 33074d5f6b8c90fb791f1838297603c6 in 905ms, sequenceid=59, compaction requested=true 2024-11-12T14:18:44,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:44,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:44,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-12T14:18:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-12T14:18:44,380 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-12T14:18:44,381 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5970 sec 2024-11-12T14:18:44,382 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 2.6020 sec 2024-11-12T14:18:44,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:44,448 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=134.18 KB heapSize=380.30 KB 2024-11-12T14:18:44,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:18:44,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:18:44,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:18:44,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:18:44,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:18:44,448 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:18:44,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/959b6757d6154f3d981d16987004e83e is 50, key is test_row_0/A:col10/1731421123829/Put/seqid=0 2024-11-12T14:18:44,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742260_1436 (size=14341) 2024-11-12T14:18:44,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421184479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:44,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:44,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:44,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:44,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421184488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:44,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:44,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:44,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:44,491 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:44,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:44,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421184489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:44,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:44,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:44,500 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:44,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:44,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38622 deadline: 1731421184499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:44,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:44,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:44,505 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:44,505 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:44,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1731421184499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:44,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:44,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:44,507 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:44,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:44,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421184599, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:44,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:44,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:44,611 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:44,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:44,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421184609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:44,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:44,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:44,618 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:44,633 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T14:18:44,835 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:44,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421184819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:44,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:44,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:44,835 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:44,839 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:44,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421184829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:44,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:44,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:44,840 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:44,857 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/959b6757d6154f3d981d16987004e83e 2024-11-12T14:18:44,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/041574a1d65242b894af49159dfa98b8 is 50, key is test_row_0/B:col10/1731421123829/Put/seqid=0 2024-11-12T14:18:44,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742261_1437 (size=12001) 2024-11-12T14:18:45,004 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:45,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421184999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:45,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:45,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:45,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:45,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:45,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38622 deadline: 1731421185019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:45,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:45,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1731421185021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:45,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:45,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:45,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:45,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:45,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:45,025 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:45,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:45,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421185149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:45,158 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:45,158 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:45,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421185149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:45,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:45,159 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:45,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:45,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:45,159 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:45,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/041574a1d65242b894af49159dfa98b8 2024-11-12T14:18:45,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/5ebb079e8d3e461ca3c37736eab690e2 is 50, key is test_row_0/C:col10/1731421123829/Put/seqid=0 2024-11-12T14:18:45,292 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742262_1438 (size=12001) 2024-11-12T14:18:45,293 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/5ebb079e8d3e461ca3c37736eab690e2 2024-11-12T14:18:45,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/959b6757d6154f3d981d16987004e83e as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/959b6757d6154f3d981d16987004e83e 2024-11-12T14:18:45,313 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/959b6757d6154f3d981d16987004e83e, entries=200, sequenceid=95, filesize=14.0 K 2024-11-12T14:18:45,314 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/041574a1d65242b894af49159dfa98b8 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/041574a1d65242b894af49159dfa98b8 2024-11-12T14:18:45,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/041574a1d65242b894af49159dfa98b8, entries=150, sequenceid=95, filesize=11.7 K 2024-11-12T14:18:45,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/5ebb079e8d3e461ca3c37736eab690e2 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/5ebb079e8d3e461ca3c37736eab690e2 2024-11-12T14:18:45,320 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/5ebb079e8d3e461ca3c37736eab690e2, entries=150, sequenceid=95, filesize=11.7 K 2024-11-12T14:18:45,321 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~380.25 KB/389376, currentSize=46.96 KB/48090 for 33074d5f6b8c90fb791f1838297603c6 in 873ms, sequenceid=95, compaction requested=true 2024-11-12T14:18:45,321 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:45,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:18:45,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:45,321 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:18:45,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:18:45,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:45,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:18:45,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:45,321 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:18:45,322 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:18:45,322 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52684 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:18:45,322 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/A is initiating minor compaction (all files) 2024-11-12T14:18:45,322 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/B is initiating minor compaction (all files) 2024-11-12T14:18:45,322 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/B in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:45,322 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/A in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:45,322 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/0b2a1dd58bdb41b6a5df074e53996cea, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/c7f6d777c75f4205812f0ab40bf47312, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/8406dac22c064be6a25f8fba349372c9, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/041574a1d65242b894af49159dfa98b8] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=46.9 K 2024-11-12T14:18:45,322 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/3c021a86f0fa48c7aead303b5250e346, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/289bb14d29db4984a404908620d4f3b2, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/95b1d08c39a54e90982b67926820508b, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/959b6757d6154f3d981d16987004e83e] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=51.4 K 2024-11-12T14:18:45,323 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c021a86f0fa48c7aead303b5250e346, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731421121790 2024-11-12T14:18:45,323 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b2a1dd58bdb41b6a5df074e53996cea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731421121790 2024-11-12T14:18:45,323 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 289bb14d29db4984a404908620d4f3b2, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731421121834 2024-11-12T14:18:45,323 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting c7f6d777c75f4205812f0ab40bf47312, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731421121855 2024-11-12T14:18:45,323 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95b1d08c39a54e90982b67926820508b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731421122543 2024-11-12T14:18:45,323 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 8406dac22c064be6a25f8fba349372c9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731421122543 2024-11-12T14:18:45,324 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 959b6757d6154f3d981d16987004e83e, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731421123829 2024-11-12T14:18:45,324 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 041574a1d65242b894af49159dfa98b8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731421123829 2024-11-12T14:18:45,331 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#A#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:45,332 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/378cd1d63cef4db184dc522c17debeee is 50, key is test_row_0/A:col10/1731421123829/Put/seqid=0 2024-11-12T14:18:45,334 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#B#compaction#361 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:45,335 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/23d7b9616a5f42a499bc88cb269f4149 is 50, key is test_row_0/B:col10/1731421123829/Put/seqid=0 2024-11-12T14:18:45,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742264_1440 (size=12139) 2024-11-12T14:18:45,350 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/23d7b9616a5f42a499bc88cb269f4149 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/23d7b9616a5f42a499bc88cb269f4149 2024-11-12T14:18:45,355 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/B of 33074d5f6b8c90fb791f1838297603c6 into 23d7b9616a5f42a499bc88cb269f4149(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:45,355 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:45,355 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/B, priority=12, startTime=1731421125321; duration=0sec 2024-11-12T14:18:45,355 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:45,355 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:B 2024-11-12T14:18:45,355 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:18:45,361 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:18:45,362 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/C is initiating minor compaction (all files) 2024-11-12T14:18:45,362 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/C in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:45,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742263_1439 (size=12139) 2024-11-12T14:18:45,362 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/f72f91d0aba34d66bef60673686bb892, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/2683857647134d86b00263ec9c80dc44, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/e3099985d2ae40b59b6a1890af491060, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/5ebb079e8d3e461ca3c37736eab690e2] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=46.9 K 2024-11-12T14:18:45,363 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting f72f91d0aba34d66bef60673686bb892, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731421121790 2024-11-12T14:18:45,363 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 2683857647134d86b00263ec9c80dc44, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731421121855 2024-11-12T14:18:45,363 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting e3099985d2ae40b59b6a1890af491060, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=59, earliestPutTs=1731421122543 2024-11-12T14:18:45,364 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ebb079e8d3e461ca3c37736eab690e2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731421123829 2024-11-12T14:18:45,371 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/378cd1d63cef4db184dc522c17debeee as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/378cd1d63cef4db184dc522c17debeee 2024-11-12T14:18:45,378 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#C#compaction#362 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:45,379 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/933ac913575946e39abefce6750352a9 is 50, key is test_row_0/C:col10/1731421123829/Put/seqid=0 2024-11-12T14:18:45,379 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/A of 33074d5f6b8c90fb791f1838297603c6 into 378cd1d63cef4db184dc522c17debeee(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:45,379 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:45,379 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/A, priority=12, startTime=1731421125321; duration=0sec 2024-11-12T14:18:45,379 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:45,379 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:A 2024-11-12T14:18:45,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742265_1441 (size=12139) 2024-11-12T14:18:45,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:45,673 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T14:18:45,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:18:45,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:45,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:18:45,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:45,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:18:45,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:45,681 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/07f0f4e2a7ff44e9a8717cd94a2624e4 is 50, key is test_row_0/A:col10/1731421125671/Put/seqid=0 2024-11-12T14:18:45,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742266_1442 (size=14341) 2024-11-12T14:18:45,703 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/07f0f4e2a7ff44e9a8717cd94a2624e4 2024-11-12T14:18:45,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/8a40607ae8084ff1a7b2c999ebcac0a4 is 50, key is test_row_0/B:col10/1731421125671/Put/seqid=0 2024-11-12T14:18:45,742 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742267_1443 (size=12001) 2024-11-12T14:18:45,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/8a40607ae8084ff1a7b2c999ebcac0a4 2024-11-12T14:18:45,755 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/d1d95649162241b8a025e1404d971fd7 is 50, key is test_row_0/C:col10/1731421125671/Put/seqid=0 2024-11-12T14:18:45,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742268_1444 (size=12001) 2024-11-12T14:18:45,768 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=108 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/d1d95649162241b8a025e1404d971fd7 2024-11-12T14:18:45,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/07f0f4e2a7ff44e9a8717cd94a2624e4 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/07f0f4e2a7ff44e9a8717cd94a2624e4 2024-11-12T14:18:45,775 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731421125717, max timestamp=1731421125773Num uniques 859; 2024-11-12T14:18:45,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731421125717, max timestamp=1731421125773Num uniques 859; 2024-11-12T14:18:45,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T14:18:45,776 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T14:18:45,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-8 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=159, cellSize=7275, totalHeapSize=16776, min timestamp=1731421125717, max timestamp=1731421125773Num uniques 859; 2024-11-12T14:18:45,778 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-8 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=7275, suffix heap size=117064,new segment heap size=16776 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=159 2024-11-12T14:18:45,782 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/07f0f4e2a7ff44e9a8717cd94a2624e4, entries=200, sequenceid=108, filesize=14.0 K 2024-11-12T14:18:45,784 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/8a40607ae8084ff1a7b2c999ebcac0a4 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/8a40607ae8084ff1a7b2c999ebcac0a4 2024-11-12T14:18:45,790 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/8a40607ae8084ff1a7b2c999ebcac0a4, entries=150, sequenceid=108, filesize=11.7 K 2024-11-12T14:18:45,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/d1d95649162241b8a025e1404d971fd7 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/d1d95649162241b8a025e1404d971fd7 2024-11-12T14:18:45,804 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/933ac913575946e39abefce6750352a9 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/933ac913575946e39abefce6750352a9 2024-11-12T14:18:45,805 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/d1d95649162241b8a025e1404d971fd7, entries=150, sequenceid=108, filesize=11.7 K 2024-11-12T14:18:45,806 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=60.38 KB/61830 for 33074d5f6b8c90fb791f1838297603c6 in 133ms, sequenceid=108, compaction requested=false 2024-11-12T14:18:45,806 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:45,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:45,808 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=67.09 KB heapSize=185.77 KB 2024-11-12T14:18:45,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:18:45,808 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:18:45,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:18:45,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:18:45,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:18:45,809 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:18:45,812 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/C of 33074d5f6b8c90fb791f1838297603c6 into 933ac913575946e39abefce6750352a9(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:45,813 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:45,813 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/C, priority=12, startTime=1731421125321; duration=0sec 2024-11-12T14:18:45,813 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:45,813 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:C 2024-11-12T14:18:45,822 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/063bc55d18c14851b945c25fe1867c31 is 50, key is test_row_0/A:col10/1731421125807/Put/seqid=0 2024-11-12T14:18:45,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742269_1445 (size=16881) 2024-11-12T14:18:45,863 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/063bc55d18c14851b945c25fe1867c31 2024-11-12T14:18:45,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/042d5d5aa0cc4a3ca15777a540dadf3d is 50, key is test_row_0/B:col10/1731421125807/Put/seqid=0 2024-11-12T14:18:45,899 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742270_1446 (size=12101) 2024-11-12T14:18:45,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-12T14:18:45,939 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:18:45,939 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:18:45,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-12T14:18:45,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T14:18:45,941 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:18:45,942 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:18:45,942 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:18:45,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/042d5d5aa0cc4a3ca15777a540dadf3d 2024-11-12T14:18:45,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/47848300f17540c28f0c0ad05370deed is 50, key is test_row_0/C:col10/1731421125807/Put/seqid=0 2024-11-12T14:18:45,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742271_1447 (size=12101) 2024-11-12T14:18:45,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:45,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421185975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:45,986 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:45,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:45,987 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:45,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:45,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421185976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:45,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:45,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:45,994 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:46,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:46,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421186009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:46,017 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_0' in 33074d5f6b8c90fb791f1838297603c6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:18:46,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:46,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:46,018 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:46,038 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:46,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38622 deadline: 1731421186029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:46,039 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_2' in 33074d5f6b8c90fb791f1838297603c6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2190 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:18:46,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:46,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:46,039 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:46,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T14:18:46,050 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:46,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1731421186039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:46,050 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_2' in 33074d5f6b8c90fb791f1838297603c6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:18:46,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:46,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:46,051 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:46,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-12T14:18:46,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:46,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:46,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:46,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:46,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:46,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:46,101 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:46,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421186089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:46,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:46,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:46,101 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:46,113 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:46,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421186099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:46,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:46,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:46,116 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:46,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-12T14:18:46,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:46,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:46,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:46,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:46,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:46,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:46,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T14:18:46,318 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:46,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421186309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:46,328 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:46,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421186319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:46,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:46,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:46,328 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:46,375 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/47848300f17540c28f0c0ad05370deed 2024-11-12T14:18:46,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/063bc55d18c14851b945c25fe1867c31 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/063bc55d18c14851b945c25fe1867c31 2024-11-12T14:18:46,383 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/063bc55d18c14851b945c25fe1867c31, entries=250, sequenceid=135, filesize=16.5 K 2024-11-12T14:18:46,383 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/042d5d5aa0cc4a3ca15777a540dadf3d as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/042d5d5aa0cc4a3ca15777a540dadf3d 2024-11-12T14:18:46,389 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/042d5d5aa0cc4a3ca15777a540dadf3d, entries=150, sequenceid=135, filesize=11.8 K 2024-11-12T14:18:46,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/47848300f17540c28f0c0ad05370deed as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/47848300f17540c28f0c0ad05370deed 2024-11-12T14:18:46,393 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/47848300f17540c28f0c0ad05370deed, entries=150, sequenceid=135, filesize=11.8 K 2024-11-12T14:18:46,394 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~185.72 KB/190176, currentSize=114.05 KB/116790 for 33074d5f6b8c90fb791f1838297603c6 in 586ms, sequenceid=135, compaction requested=true 2024-11-12T14:18:46,394 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:46,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:18:46,394 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:46,394 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:46,394 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:46,395 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43361 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:46,395 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/A is initiating minor compaction (all files) 2024-11-12T14:18:46,395 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/A in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:46,395 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/378cd1d63cef4db184dc522c17debeee, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/07f0f4e2a7ff44e9a8717cd94a2624e4, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/063bc55d18c14851b945c25fe1867c31] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=42.3 K 2024-11-12T14:18:46,395 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36241 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:46,395 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/B is initiating minor compaction (all files) 2024-11-12T14:18:46,395 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/B in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:46,395 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/23d7b9616a5f42a499bc88cb269f4149, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/8a40607ae8084ff1a7b2c999ebcac0a4, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/042d5d5aa0cc4a3ca15777a540dadf3d] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=35.4 K 2024-11-12T14:18:46,396 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 378cd1d63cef4db184dc522c17debeee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731421123829 2024-11-12T14:18:46,396 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 23d7b9616a5f42a499bc88cb269f4149, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731421123829 2024-11-12T14:18:46,396 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:18:46,396 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 07f0f4e2a7ff44e9a8717cd94a2624e4, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731421124474 2024-11-12T14:18:46,396 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a40607ae8084ff1a7b2c999ebcac0a4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731421124486 2024-11-12T14:18:46,397 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 063bc55d18c14851b945c25fe1867c31, keycount=250, bloomtype=ROW, size=16.5 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731421125773 2024-11-12T14:18:46,397 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 042d5d5aa0cc4a3ca15777a540dadf3d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731421125773 2024-11-12T14:18:46,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:46,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:18:46,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:46,404 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#A#compaction#369 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:46,404 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#B#compaction#370 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:46,404 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/7c653b9d40b946ec82cfc45b65832caa is 50, key is test_row_0/A:col10/1731421125807/Put/seqid=0 2024-11-12T14:18:46,404 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/faf6f92af730404580a80027a7735e7d is 50, key is test_row_0/B:col10/1731421125807/Put/seqid=0 2024-11-12T14:18:46,406 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-12T14:18:46,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:46,407 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T14:18:46,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:18:46,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:46,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:18:46,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:46,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:18:46,407 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:46,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/5175e3a00a33414b99a15cc5fdf2f2d2 is 50, key is test_row_0/A:col10/1731421125973/Put/seqid=0 2024-11-12T14:18:46,433 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742272_1448 (size=12341) 2024-11-12T14:18:46,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742273_1449 (size=12341) 2024-11-12T14:18:46,447 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/faf6f92af730404580a80027a7735e7d as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/faf6f92af730404580a80027a7735e7d 2024-11-12T14:18:46,448 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/7c653b9d40b946ec82cfc45b65832caa as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/7c653b9d40b946ec82cfc45b65832caa 2024-11-12T14:18:46,454 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/B of 33074d5f6b8c90fb791f1838297603c6 into faf6f92af730404580a80027a7735e7d(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:46,454 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/A of 33074d5f6b8c90fb791f1838297603c6 into 7c653b9d40b946ec82cfc45b65832caa(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:46,454 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:46,454 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:46,454 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/B, priority=13, startTime=1731421126394; duration=0sec 2024-11-12T14:18:46,454 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/A, priority=13, startTime=1731421126394; duration=0sec 2024-11-12T14:18:46,455 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:46,455 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:B 2024-11-12T14:18:46,455 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:46,455 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:A 2024-11-12T14:18:46,455 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:46,455 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36241 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:46,455 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/C is initiating minor compaction (all files) 2024-11-12T14:18:46,455 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/C in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:46,455 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/933ac913575946e39abefce6750352a9, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/d1d95649162241b8a025e1404d971fd7, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/47848300f17540c28f0c0ad05370deed] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=35.4 K 2024-11-12T14:18:46,456 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 933ac913575946e39abefce6750352a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731421123829 2024-11-12T14:18:46,456 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting d1d95649162241b8a025e1404d971fd7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=108, earliestPutTs=1731421124486 2024-11-12T14:18:46,456 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 47848300f17540c28f0c0ad05370deed, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731421125773 2024-11-12T14:18:46,461 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742274_1450 (size=12151) 2024-11-12T14:18:46,463 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#C#compaction#372 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:46,463 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/e0a17bdbaf594b1ea86283684879f2db is 50, key is test_row_0/C:col10/1731421125807/Put/seqid=0 2024-11-12T14:18:46,466 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742275_1451 (size=12341) 2024-11-12T14:18:46,469 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/5175e3a00a33414b99a15cc5fdf2f2d2 2024-11-12T14:18:46,472 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/e0a17bdbaf594b1ea86283684879f2db as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/e0a17bdbaf594b1ea86283684879f2db 2024-11-12T14:18:46,481 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/C of 33074d5f6b8c90fb791f1838297603c6 into e0a17bdbaf594b1ea86283684879f2db(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:46,481 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:46,481 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/C, priority=13, startTime=1731421126400; duration=0sec 2024-11-12T14:18:46,481 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:46,481 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:C 2024-11-12T14:18:46,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/97765a8484db45ea9b0481be18ed9295 is 50, key is test_row_0/B:col10/1731421125973/Put/seqid=0 2024-11-12T14:18:46,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742276_1452 (size=12151) 2024-11-12T14:18:46,495 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/97765a8484db45ea9b0481be18ed9295 2024-11-12T14:18:46,503 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/d49edaafe55f46b3a550e5b18f61a5c0 is 50, key is test_row_0/C:col10/1731421125973/Put/seqid=0 2024-11-12T14:18:46,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742277_1453 (size=12151) 2024-11-12T14:18:46,517 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=156 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/d49edaafe55f46b3a550e5b18f61a5c0 2024-11-12T14:18:46,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/5175e3a00a33414b99a15cc5fdf2f2d2 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/5175e3a00a33414b99a15cc5fdf2f2d2 2024-11-12T14:18:46,523 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/5175e3a00a33414b99a15cc5fdf2f2d2, entries=150, sequenceid=156, filesize=11.9 K 2024-11-12T14:18:46,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/97765a8484db45ea9b0481be18ed9295 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/97765a8484db45ea9b0481be18ed9295 2024-11-12T14:18:46,527 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/97765a8484db45ea9b0481be18ed9295, entries=150, sequenceid=156, filesize=11.9 K 2024-11-12T14:18:46,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/d49edaafe55f46b3a550e5b18f61a5c0 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/d49edaafe55f46b3a550e5b18f61a5c0 2024-11-12T14:18:46,530 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/d49edaafe55f46b3a550e5b18f61a5c0, entries=150, sequenceid=156, filesize=11.9 K 2024-11-12T14:18:46,531 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=0 B/0 for 33074d5f6b8c90fb791f1838297603c6 in 124ms, sequenceid=156, compaction requested=false 2024-11-12T14:18:46,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:46,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:46,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-12T14:18:46,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-12T14:18:46,533 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-12T14:18:46,533 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 590 msec 2024-11-12T14:18:46,537 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 595 msec 2024-11-12T14:18:46,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-12T14:18:46,569 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:18:46,569 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:18:46,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees 2024-11-12T14:18:46,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-12T14:18:46,571 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:18:46,572 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=113, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:18:46,572 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:18:46,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:46,653 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T14:18:46,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:18:46,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:46,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:18:46,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:46,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:18:46,653 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:46,657 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/17f07ef41935475fbee71e8cf8bf6990 is 50, key is test_row_0/A:col10/1731421126644/Put/seqid=0 2024-11-12T14:18:46,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-12T14:18:46,713 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742278_1454 (size=14541) 2024-11-12T14:18:46,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/17f07ef41935475fbee71e8cf8bf6990 2024-11-12T14:18:46,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-12T14:18:46,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:46,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:46,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:46,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:46,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:46,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:46,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/6ca8c2e9ffee43a2bc752db1b7c25483 is 50, key is test_row_0/B:col10/1731421126644/Put/seqid=0 2024-11-12T14:18:46,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742279_1455 (size=12151) 2024-11-12T14:18:46,764 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-7 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731421126673, max timestamp=1731421126759Num uniques 859; 2024-11-12T14:18:46,765 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-7 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T14:18:46,765 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-5 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731421126673, max timestamp=1731421126759Num uniques 859; 2024-11-12T14:18:46,765 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-5 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T14:18:46,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=150, cellSize=6870, totalHeapSize=15840, min timestamp=1731421126673, max timestamp=1731421126759Num uniques 859; 2024-11-12T14:18:46,766 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=6870, suffix heap size=117064,new segment heap size=15840 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=150 2024-11-12T14:18:46,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-12T14:18:46,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:46,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:46,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:46,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:46,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:46,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:46,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:46,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421186870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:46,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:46,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421186870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:46,881 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:46,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:46,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:46,882 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:46,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-12T14:18:46,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:47,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421186989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:47,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:47,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:47,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:47,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421186990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:47,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:47,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:47,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:47,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:47,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-12T14:18:47,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:47,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:47,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:47,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:47,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:47,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:47,161 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/6ca8c2e9ffee43a2bc752db1b7c25483 2024-11-12T14:18:47,175 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/d4716fd012c046f2a953a87c8ff0fe33 is 50, key is test_row_0/C:col10/1731421126644/Put/seqid=0 2024-11-12T14:18:47,186 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-12T14:18:47,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:47,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:47,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:47,187 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:47,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:47,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:47,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742280_1456 (size=12151) 2024-11-12T14:18:47,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-12T14:18:47,214 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:47,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421187213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:47,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:47,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:47,216 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:47,230 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:47,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421187216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:47,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:47,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:47,230 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:47,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-12T14:18:47,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:47,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:47,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:47,340 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:47,340 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:47,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:47,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-12T14:18:47,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:47,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:47,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:47,496 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] handler.RSProcedureHandler(58): pid=114 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:47,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=114 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:47,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=114 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:47,533 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:47,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421187522, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:47,534 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:47,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:47,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421187539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:47,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:47,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:47,554 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:47,589 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/d4716fd012c046f2a953a87c8ff0fe33 2024-11-12T14:18:47,593 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/17f07ef41935475fbee71e8cf8bf6990 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/17f07ef41935475fbee71e8cf8bf6990 2024-11-12T14:18:47,598 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/17f07ef41935475fbee71e8cf8bf6990, entries=200, sequenceid=169, filesize=14.2 K 2024-11-12T14:18:47,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/6ca8c2e9ffee43a2bc752db1b7c25483 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/6ca8c2e9ffee43a2bc752db1b7c25483 2024-11-12T14:18:47,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/6ca8c2e9ffee43a2bc752db1b7c25483, entries=150, sequenceid=169, filesize=11.9 K 2024-11-12T14:18:47,603 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/d4716fd012c046f2a953a87c8ff0fe33 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/d4716fd012c046f2a953a87c8ff0fe33 2024-11-12T14:18:47,615 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/d4716fd012c046f2a953a87c8ff0fe33, entries=150, sequenceid=169, filesize=11.9 K 2024-11-12T14:18:47,616 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=131.55 KB/134705 for 33074d5f6b8c90fb791f1838297603c6 in 964ms, sequenceid=169, compaction requested=true 2024-11-12T14:18:47,616 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:47,616 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:47,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:18:47,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:47,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:18:47,616 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:47,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:47,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:18:47,616 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:47,617 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39033 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:47,617 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/A is initiating minor compaction (all files) 2024-11-12T14:18:47,617 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/A in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:47,617 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/7c653b9d40b946ec82cfc45b65832caa, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/5175e3a00a33414b99a15cc5fdf2f2d2, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/17f07ef41935475fbee71e8cf8bf6990] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=38.1 K 2024-11-12T14:18:47,618 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c653b9d40b946ec82cfc45b65832caa, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731421125773 2024-11-12T14:18:47,618 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:47,618 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/B is initiating minor compaction (all files) 2024-11-12T14:18:47,618 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/B in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:47,618 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/faf6f92af730404580a80027a7735e7d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/97765a8484db45ea9b0481be18ed9295, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/6ca8c2e9ffee43a2bc752db1b7c25483] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=35.8 K 2024-11-12T14:18:47,619 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5175e3a00a33414b99a15cc5fdf2f2d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731421125928 2024-11-12T14:18:47,619 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17f07ef41935475fbee71e8cf8bf6990, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731421126632 2024-11-12T14:18:47,619 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting faf6f92af730404580a80027a7735e7d, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731421125773 2024-11-12T14:18:47,620 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 97765a8484db45ea9b0481be18ed9295, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731421125928 2024-11-12T14:18:47,621 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ca8c2e9ffee43a2bc752db1b7c25483, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731421126632 2024-11-12T14:18:47,629 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#A#compaction#378 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:47,629 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/ed4ef2d5d0da43229322bd8d80d6f850 is 50, key is test_row_0/A:col10/1731421126644/Put/seqid=0 2024-11-12T14:18:47,636 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#B#compaction#379 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:47,636 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/a7e2196866bb478eae731f18dcc6cf55 is 50, key is test_row_0/B:col10/1731421126644/Put/seqid=0 2024-11-12T14:18:47,653 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=114 2024-11-12T14:18:47,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:47,653 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=131.55 KB heapSize=374.30 KB 2024-11-12T14:18:47,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:18:47,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:18:47,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:18:47,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:18:47,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:18:47,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:18:47,674 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742281_1457 (size=12493) 2024-11-12T14:18:47,702 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742282_1458 (size=12493) 2024-11-12T14:18:47,706 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/a7e2196866bb478eae731f18dcc6cf55 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/a7e2196866bb478eae731f18dcc6cf55 2024-11-12T14:18:47,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/c401cb2cbd974ae0b01845d6fd2670b1 is 50, key is test_row_0/A:col10/1731421126869/Put/seqid=0 2024-11-12T14:18:47,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-12T14:18:47,716 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/B of 33074d5f6b8c90fb791f1838297603c6 into a7e2196866bb478eae731f18dcc6cf55(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:47,716 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:47,717 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/B, priority=13, startTime=1731421127616; duration=0sec 2024-11-12T14:18:47,717 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:47,717 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:B 2024-11-12T14:18:47,717 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:47,718 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36643 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:47,718 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/C is initiating minor compaction (all files) 2024-11-12T14:18:47,718 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/C in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:47,718 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/e0a17bdbaf594b1ea86283684879f2db, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/d49edaafe55f46b3a550e5b18f61a5c0, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/d4716fd012c046f2a953a87c8ff0fe33] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=35.8 K 2024-11-12T14:18:47,719 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting e0a17bdbaf594b1ea86283684879f2db, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1731421125773 2024-11-12T14:18:47,719 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting d49edaafe55f46b3a550e5b18f61a5c0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=156, earliestPutTs=1731421125928 2024-11-12T14:18:47,719 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting d4716fd012c046f2a953a87c8ff0fe33, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731421126632 2024-11-12T14:18:47,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742283_1459 (size=12151) 2024-11-12T14:18:47,728 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/c401cb2cbd974ae0b01845d6fd2670b1 2024-11-12T14:18:47,742 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#C#compaction#381 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:47,743 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/825d0278cec34f4eae3f38d48c7880c4 is 50, key is test_row_0/C:col10/1731421126644/Put/seqid=0 2024-11-12T14:18:47,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/c9ba1bb4261349b2bb1b203a30ae6e7f is 50, key is test_row_0/B:col10/1731421126869/Put/seqid=0 2024-11-12T14:18:47,752 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742285_1461 (size=12151) 2024-11-12T14:18:47,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742284_1460 (size=12493) 2024-11-12T14:18:47,800 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/825d0278cec34f4eae3f38d48c7880c4 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/825d0278cec34f4eae3f38d48c7880c4 2024-11-12T14:18:47,808 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/C of 33074d5f6b8c90fb791f1838297603c6 into 825d0278cec34f4eae3f38d48c7880c4(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:47,808 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:47,809 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/C, priority=13, startTime=1731421127616; duration=0sec 2024-11-12T14:18:47,809 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:47,809 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:C 2024-11-12T14:18:48,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:48,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:48,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:48,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421188068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:48,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:48,077 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/ed4ef2d5d0da43229322bd8d80d6f850 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/ed4ef2d5d0da43229322bd8d80d6f850 2024-11-12T14:18:48,081 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/A of 33074d5f6b8c90fb791f1838297603c6 into ed4ef2d5d0da43229322bd8d80d6f850(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:48,081 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:48,081 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/A, priority=13, startTime=1731421127616; duration=0sec 2024-11-12T14:18:48,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:48,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421188069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:48,081 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:48,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421188069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:48,081 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:48,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38622 deadline: 1731421188070, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:48,082 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:48,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1731421188074, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:48,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,083 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:48,083 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,084 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:48,084 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:48,084 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:A 2024-11-12T14:18:48,152 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/c9ba1bb4261349b2bb1b203a30ae6e7f 2024-11-12T14:18:48,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/b710ef911d71482ca4f8ea5a406440f2 is 50, key is test_row_0/C:col10/1731421126869/Put/seqid=0 2024-11-12T14:18:48,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742286_1462 (size=12151) 2024-11-12T14:18:48,171 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.09 KB at sequenceid=205 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/b710ef911d71482ca4f8ea5a406440f2 2024-11-12T14:18:48,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/c401cb2cbd974ae0b01845d6fd2670b1 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/c401cb2cbd974ae0b01845d6fd2670b1 2024-11-12T14:18:48,180 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/c401cb2cbd974ae0b01845d6fd2670b1, entries=150, sequenceid=205, filesize=11.9 K 2024-11-12T14:18:48,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/c9ba1bb4261349b2bb1b203a30ae6e7f as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/c9ba1bb4261349b2bb1b203a30ae6e7f 2024-11-12T14:18:48,184 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/c9ba1bb4261349b2bb1b203a30ae6e7f, entries=150, sequenceid=205, filesize=11.9 K 2024-11-12T14:18:48,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/b710ef911d71482ca4f8ea5a406440f2 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/b710ef911d71482ca4f8ea5a406440f2 2024-11-12T14:18:48,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:48,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421188179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:48,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,188 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:48,189 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/b710ef911d71482ca4f8ea5a406440f2, entries=150, sequenceid=205, filesize=11.9 K 2024-11-12T14:18:48,189 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~131.55 KB/134705, heapSize ~374.26 KB/383240, currentSize=46.96 KB/48090 for 33074d5f6b8c90fb791f1838297603c6 in 536ms, sequenceid=205, compaction requested=false 2024-11-12T14:18:48,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.HRegion(2603): Flush status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:48,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:48,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=114}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=114 2024-11-12T14:18:48,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=114 2024-11-12T14:18:48,192 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-12T14:18:48,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:48,193 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6190 sec 2024-11-12T14:18:48,193 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T14:18:48,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:18:48,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:48,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:18:48,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:48,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:18:48,193 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:48,194 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=113, table=TestAcidGuarantees in 1.6230 sec 2024-11-12T14:18:48,197 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/dae343df2400410d9f133575924dcdcd is 50, key is test_row_0/A:col10/1731421128191/Put/seqid=0 2024-11-12T14:18:48,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742287_1463 (size=14541) 2024-11-12T14:18:48,226 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/dae343df2400410d9f133575924dcdcd 2024-11-12T14:18:48,232 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/a773a506c46e4fb9839ef2551fd20583 is 50, key is test_row_0/B:col10/1731421128191/Put/seqid=0 2024-11-12T14:18:48,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731421128221, max timestamp=1731421128243Num uniques 859; 2024-11-12T14:18:48,246 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T14:18:48,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731421128221, max timestamp=1731421128243Num uniques 859; 2024-11-12T14:18:48,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T14:18:48,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731421128221, max timestamp=1731421128243Num uniques 859; 2024-11-12T14:18:48,248 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T14:18:48,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742288_1464 (size=12151) 2024-11-12T14:18:48,251 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/a773a506c46e4fb9839ef2551fd20583 2024-11-12T14:18:48,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/cdca18a5fc16438592e1bbe1c675d62d is 50, key is test_row_0/C:col10/1731421128191/Put/seqid=0 2024-11-12T14:18:48,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742289_1465 (size=12151) 2024-11-12T14:18:48,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/cdca18a5fc16438592e1bbe1c675d62d 2024-11-12T14:18:48,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/dae343df2400410d9f133575924dcdcd as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/dae343df2400410d9f133575924dcdcd 2024-11-12T14:18:48,271 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/dae343df2400410d9f133575924dcdcd, entries=200, sequenceid=219, filesize=14.2 K 2024-11-12T14:18:48,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/a773a506c46e4fb9839ef2551fd20583 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/a773a506c46e4fb9839ef2551fd20583 2024-11-12T14:18:48,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/a773a506c46e4fb9839ef2551fd20583, entries=150, sequenceid=219, filesize=11.9 K 2024-11-12T14:18:48,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/cdca18a5fc16438592e1bbe1c675d62d as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/cdca18a5fc16438592e1bbe1c675d62d 2024-11-12T14:18:48,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/cdca18a5fc16438592e1bbe1c675d62d, entries=150, sequenceid=219, filesize=11.9 K 2024-11-12T14:18:48,283 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=87.22 KB/89310 for 33074d5f6b8c90fb791f1838297603c6 in 90ms, sequenceid=219, compaction requested=true 2024-11-12T14:18:48,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:48,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:18:48,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:48,283 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:48,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:18:48,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:48,283 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:48,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:18:48,283 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:48,284 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:48,284 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:48,284 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/A is initiating minor compaction (all files) 2024-11-12T14:18:48,284 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/B is initiating minor compaction (all files) 2024-11-12T14:18:48,284 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/B in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:48,284 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/a7e2196866bb478eae731f18dcc6cf55, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/c9ba1bb4261349b2bb1b203a30ae6e7f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/a773a506c46e4fb9839ef2551fd20583] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=35.9 K 2024-11-12T14:18:48,284 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/A in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:48,284 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/ed4ef2d5d0da43229322bd8d80d6f850, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/c401cb2cbd974ae0b01845d6fd2670b1, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/dae343df2400410d9f133575924dcdcd] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=38.3 K 2024-11-12T14:18:48,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:48,286 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=93.93 KB heapSize=256.08 KB 2024-11-12T14:18:48,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:18:48,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:18:48,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:18:48,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:18:48,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:18:48,286 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:18:48,287 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting a7e2196866bb478eae731f18dcc6cf55, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731421126632 2024-11-12T14:18:48,288 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting ed4ef2d5d0da43229322bd8d80d6f850, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731421126632 2024-11-12T14:18:48,288 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting c9ba1bb4261349b2bb1b203a30ae6e7f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731421126824 2024-11-12T14:18:48,288 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting c401cb2cbd974ae0b01845d6fd2670b1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731421126824 2024-11-12T14:18:48,288 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting a773a506c46e4fb9839ef2551fd20583, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731421128067 2024-11-12T14:18:48,288 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting dae343df2400410d9f133575924dcdcd, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731421128052 2024-11-12T14:18:48,292 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/1105bee6c0fd40aca38f76a91d225adc is 50, key is test_row_0/A:col10/1731421128277/Put/seqid=0 2024-11-12T14:18:48,297 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#B#compaction#388 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:48,297 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/f8d75caced5e4b85bcd63bfba79a00ef is 50, key is test_row_0/B:col10/1731421128191/Put/seqid=0 2024-11-12T14:18:48,306 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#A#compaction#389 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:48,306 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/a7b31d0bdde94f908877cb0e876270b6 is 50, key is test_row_0/A:col10/1731421128191/Put/seqid=0 2024-11-12T14:18:48,321 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742290_1466 (size=14541) 2024-11-12T14:18:48,322 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/1105bee6c0fd40aca38f76a91d225adc 2024-11-12T14:18:48,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742291_1467 (size=12595) 2024-11-12T14:18:48,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742292_1468 (size=12595) 2024-11-12T14:18:48,342 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/a7b31d0bdde94f908877cb0e876270b6 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/a7b31d0bdde94f908877cb0e876270b6 2024-11-12T14:18:48,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/0e52e25c0d4e4104964c7b48db5865dc is 50, key is test_row_0/B:col10/1731421128277/Put/seqid=0 2024-11-12T14:18:48,348 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/A of 33074d5f6b8c90fb791f1838297603c6 into a7b31d0bdde94f908877cb0e876270b6(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:48,348 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:48,348 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/A, priority=13, startTime=1731421128283; duration=0sec 2024-11-12T14:18:48,348 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:48,348 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:A 2024-11-12T14:18:48,348 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:48,351 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:48,351 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/C is initiating minor compaction (all files) 2024-11-12T14:18:48,351 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/C in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:48,351 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742293_1469 (size=12151) 2024-11-12T14:18:48,352 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/825d0278cec34f4eae3f38d48c7880c4, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/b710ef911d71482ca4f8ea5a406440f2, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/cdca18a5fc16438592e1bbe1c675d62d] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=35.9 K 2024-11-12T14:18:48,352 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/0e52e25c0d4e4104964c7b48db5865dc 2024-11-12T14:18:48,352 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 825d0278cec34f4eae3f38d48c7880c4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731421126632 2024-11-12T14:18:48,352 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b710ef911d71482ca4f8ea5a406440f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=205, earliestPutTs=1731421126824 2024-11-12T14:18:48,353 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdca18a5fc16438592e1bbe1c675d62d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731421128067 2024-11-12T14:18:48,363 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/40004a1ec9ee43e78911a97ce7e24293 is 50, key is test_row_0/C:col10/1731421128277/Put/seqid=0 2024-11-12T14:18:48,367 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#C#compaction#392 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:48,367 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/9e5c8472b4174a8680b1b1e5f035f1ec is 50, key is test_row_0/C:col10/1731421128191/Put/seqid=0 2024-11-12T14:18:48,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:48,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421188359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:48,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:48,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:48,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38622 deadline: 1731421188359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:48,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,380 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:48,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:48,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1731421188359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:48,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,380 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:48,409 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:48,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421188399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:48,410 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,411 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:48,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742295_1471 (size=12595) 2024-11-12T14:18:48,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742294_1470 (size=12151) 2024-11-12T14:18:48,424 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=248 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/40004a1ec9ee43e78911a97ce7e24293 2024-11-12T14:18:48,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/1105bee6c0fd40aca38f76a91d225adc as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/1105bee6c0fd40aca38f76a91d225adc 2024-11-12T14:18:48,443 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/9e5c8472b4174a8680b1b1e5f035f1ec as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/9e5c8472b4174a8680b1b1e5f035f1ec 2024-11-12T14:18:48,447 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/1105bee6c0fd40aca38f76a91d225adc, entries=200, sequenceid=248, filesize=14.2 K 2024-11-12T14:18:48,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/0e52e25c0d4e4104964c7b48db5865dc as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/0e52e25c0d4e4104964c7b48db5865dc 2024-11-12T14:18:48,451 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/C of 33074d5f6b8c90fb791f1838297603c6 into 9e5c8472b4174a8680b1b1e5f035f1ec(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:48,451 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:48,452 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/C, priority=13, startTime=1731421128283; duration=0sec 2024-11-12T14:18:48,452 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:48,452 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:C 2024-11-12T14:18:48,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/0e52e25c0d4e4104964c7b48db5865dc, entries=150, sequenceid=248, filesize=11.9 K 2024-11-12T14:18:48,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/40004a1ec9ee43e78911a97ce7e24293 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/40004a1ec9ee43e78911a97ce7e24293 2024-11-12T14:18:48,461 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/40004a1ec9ee43e78911a97ce7e24293, entries=150, sequenceid=248, filesize=11.9 K 2024-11-12T14:18:48,464 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~256.03 KB/262176, currentSize=93.93 KB/96180 for 33074d5f6b8c90fb791f1838297603c6 in 178ms, sequenceid=248, compaction requested=false 2024-11-12T14:18:48,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:48,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:48,499 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T14:18:48,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:18:48,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:48,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:18:48,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:48,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:18:48,499 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:48,506 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/d70a7cb9d7e043ef9d49c27a6abac988 is 50, key is test_row_0/A:col10/1731421128495/Put/seqid=0 2024-11-12T14:18:48,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742296_1472 (size=14741) 2024-11-12T14:18:48,544 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/d70a7cb9d7e043ef9d49c27a6abac988 2024-11-12T14:18:48,553 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/6d20ecb93ed1404bb9a6844e01db91f6 is 50, key is test_row_0/B:col10/1731421128495/Put/seqid=0 2024-11-12T14:18:48,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:48,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1731421188543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:48,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:48,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,561 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:48,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421188556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:48,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,562 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:48,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,562 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:48,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38622 deadline: 1731421188556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:48,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,562 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:48,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742297_1473 (size=12301) 2024-11-12T14:18:48,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/6d20ecb93ed1404bb9a6844e01db91f6 2024-11-12T14:18:48,591 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/3ecbc83fffcb48eabcceceef22317535 is 50, key is test_row_0/C:col10/1731421128495/Put/seqid=0 2024-11-12T14:18:48,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742298_1474 (size=12301) 2024-11-12T14:18:48,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:48,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421188670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:48,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:48,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1731421188670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:48,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:48,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,680 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:48,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:48,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38622 deadline: 1731421188670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:48,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:48,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-12T14:18:48,721 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:18:48,721 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:18:48,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees 2024-11-12T14:18:48,724 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:18:48,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-12T14:18:48,726 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=115, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:18:48,726 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:18:48,737 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/f8d75caced5e4b85bcd63bfba79a00ef as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/f8d75caced5e4b85bcd63bfba79a00ef 2024-11-12T14:18:48,742 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:48,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421188727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:48,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,743 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:48,744 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/B of 33074d5f6b8c90fb791f1838297603c6 into f8d75caced5e4b85bcd63bfba79a00ef(size=12.3 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:48,744 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:48,744 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/B, priority=13, startTime=1731421128283; duration=0sec 2024-11-12T14:18:48,744 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:48,744 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:B 2024-11-12T14:18:48,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-12T14:18:48,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-12T14:18:48,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:48,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:48,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:48,879 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:48,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:48,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:48,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:48,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421188890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:48,898 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,899 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:48,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1731421188892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:48,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,906 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:48,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:48,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38622 deadline: 1731421188893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:48,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:48,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:49,013 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/3ecbc83fffcb48eabcceceef22317535 2024-11-12T14:18:49,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/d70a7cb9d7e043ef9d49c27a6abac988 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/d70a7cb9d7e043ef9d49c27a6abac988 2024-11-12T14:18:49,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/d70a7cb9d7e043ef9d49c27a6abac988, entries=200, sequenceid=268, filesize=14.4 K 2024-11-12T14:18:49,031 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/6d20ecb93ed1404bb9a6844e01db91f6 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/6d20ecb93ed1404bb9a6844e01db91f6 2024-11-12T14:18:49,033 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-12T14:18:49,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:49,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:49,036 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/6d20ecb93ed1404bb9a6844e01db91f6, entries=150, sequenceid=268, filesize=12.0 K 2024-11-12T14:18:49,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:49,036 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:49,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:49,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/3ecbc83fffcb48eabcceceef22317535 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/3ecbc83fffcb48eabcceceef22317535 2024-11-12T14:18:49,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:49,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-12T14:18:49,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/3ecbc83fffcb48eabcceceef22317535, entries=150, sequenceid=268, filesize=12.0 K 2024-11-12T14:18:49,053 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for 33074d5f6b8c90fb791f1838297603c6 in 554ms, sequenceid=268, compaction requested=true 2024-11-12T14:18:49,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:49,053 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:49,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:18:49,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:49,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:18:49,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:49,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:18:49,054 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T14:18:49,054 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41877 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:49,054 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/A is initiating minor compaction (all files) 2024-11-12T14:18:49,055 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/A in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:49,055 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/a7b31d0bdde94f908877cb0e876270b6, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/1105bee6c0fd40aca38f76a91d225adc, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/d70a7cb9d7e043ef9d49c27a6abac988] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=40.9 K 2024-11-12T14:18:49,055 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting a7b31d0bdde94f908877cb0e876270b6, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731421128067 2024-11-12T14:18:49,056 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 1105bee6c0fd40aca38f76a91d225adc, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731421128258 2024-11-12T14:18:49,056 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:49,057 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting d70a7cb9d7e043ef9d49c27a6abac988, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731421128328 2024-11-12T14:18:49,068 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:49,068 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/B is initiating minor compaction (all files) 2024-11-12T14:18:49,068 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/B in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:49,068 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/f8d75caced5e4b85bcd63bfba79a00ef, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/0e52e25c0d4e4104964c7b48db5865dc, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/6d20ecb93ed1404bb9a6844e01db91f6] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=36.2 K 2024-11-12T14:18:49,072 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#A#compaction#396 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:49,072 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8d75caced5e4b85bcd63bfba79a00ef, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731421128067 2024-11-12T14:18:49,074 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/96b7b8eae907406c9e117f29a8d82dea is 50, key is test_row_0/A:col10/1731421128495/Put/seqid=0 2024-11-12T14:18:49,075 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0e52e25c0d4e4104964c7b48db5865dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731421128277 2024-11-12T14:18:49,078 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6d20ecb93ed1404bb9a6844e01db91f6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731421128328 2024-11-12T14:18:49,102 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#B#compaction#397 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:49,102 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/2a4958c66c1f42a4a0c84381b9ab9af9 is 50, key is test_row_0/B:col10/1731421128495/Put/seqid=0 2024-11-12T14:18:49,109 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=87.22 KB heapSize=259.73 KB 2024-11-12T14:18:49,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:18:49,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:49,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:18:49,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:49,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:18:49,109 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:49,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:49,125 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742299_1475 (size=12847) 2024-11-12T14:18:49,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742300_1476 (size=12847) 2024-11-12T14:18:49,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/5ef3858cda4c4a9891362161cc9146f2 is 50, key is test_row_0/A:col10/1731421128553/Put/seqid=0 2024-11-12T14:18:49,172 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/96b7b8eae907406c9e117f29a8d82dea as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/96b7b8eae907406c9e117f29a8d82dea 2024-11-12T14:18:49,186 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/A of 33074d5f6b8c90fb791f1838297603c6 into 96b7b8eae907406c9e117f29a8d82dea(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:49,186 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:49,186 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/A, priority=13, startTime=1731421129053; duration=0sec 2024-11-12T14:18:49,186 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:49,186 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:A 2024-11-12T14:18:49,186 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:49,188 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37047 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:49,188 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/C is initiating minor compaction (all files) 2024-11-12T14:18:49,188 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/C in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:49,188 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/9e5c8472b4174a8680b1b1e5f035f1ec, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/40004a1ec9ee43e78911a97ce7e24293, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/3ecbc83fffcb48eabcceceef22317535] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=36.2 K 2024-11-12T14:18:49,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742301_1477 (size=14741) 2024-11-12T14:18:49,189 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e5c8472b4174a8680b1b1e5f035f1ec, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1731421128067 2024-11-12T14:18:49,189 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/5ef3858cda4c4a9891362161cc9146f2 2024-11-12T14:18:49,190 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 40004a1ec9ee43e78911a97ce7e24293, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=248, earliestPutTs=1731421128277 2024-11-12T14:18:49,190 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ecbc83fffcb48eabcceceef22317535, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731421128328 2024-11-12T14:18:49,192 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/2a4958c66c1f42a4a0c84381b9ab9af9 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/2a4958c66c1f42a4a0c84381b9ab9af9 2024-11-12T14:18:49,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/cd6d782800df458ca7c6ad5f86fbf4d8 is 50, key is test_row_0/B:col10/1731421128553/Put/seqid=0 2024-11-12T14:18:49,204 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-12T14:18:49,204 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:49,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:49,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:49,205 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:49,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:49,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:49,214 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#C#compaction#400 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:49,215 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/42f383bccb12469ea531d35295dd3adf is 50, key is test_row_0/C:col10/1731421128495/Put/seqid=0 2024-11-12T14:18:49,224 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/B of 33074d5f6b8c90fb791f1838297603c6 into 2a4958c66c1f42a4a0c84381b9ab9af9(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:49,224 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:49,224 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/B, priority=13, startTime=1731421129054; duration=0sec 2024-11-12T14:18:49,224 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:49,224 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:B 2024-11-12T14:18:49,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742302_1478 (size=12301) 2024-11-12T14:18:49,248 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/cd6d782800df458ca7c6ad5f86fbf4d8 2024-11-12T14:18:49,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742303_1479 (size=12847) 2024-11-12T14:18:49,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:49,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421189253, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:49,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,284 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:49,291 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/42f383bccb12469ea531d35295dd3adf as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/42f383bccb12469ea531d35295dd3adf 2024-11-12T14:18:49,295 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/52b64e4c174a45e38dc3b6a9f77ffb1e is 50, key is test_row_0/C:col10/1731421128553/Put/seqid=0 2024-11-12T14:18:49,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:49,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421189273, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:49,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:49,299 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:49,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421189277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:49,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,301 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:49,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:49,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1731421189280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:49,302 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38622 deadline: 1731421189285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:49,302 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:49,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,303 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:49,316 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/C of 33074d5f6b8c90fb791f1838297603c6 into 42f383bccb12469ea531d35295dd3adf(size=12.5 K), total size for store is 12.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:49,316 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:49,316 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/C, priority=13, startTime=1731421129054; duration=0sec 2024-11-12T14:18:49,316 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:49,316 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:C 2024-11-12T14:18:49,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742304_1480 (size=12301) 2024-11-12T14:18:49,332 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/52b64e4c174a45e38dc3b6a9f77ffb1e 2024-11-12T14:18:49,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-12T14:18:49,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-12T14:18:49,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:49,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:49,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:49,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:49,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:49,366 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/5ef3858cda4c4a9891362161cc9146f2 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/5ef3858cda4c4a9891362161cc9146f2 2024-11-12T14:18:49,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:49,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/5ef3858cda4c4a9891362161cc9146f2, entries=200, sequenceid=285, filesize=14.4 K 2024-11-12T14:18:49,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/cd6d782800df458ca7c6ad5f86fbf4d8 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/cd6d782800df458ca7c6ad5f86fbf4d8 2024-11-12T14:18:49,398 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/cd6d782800df458ca7c6ad5f86fbf4d8, entries=150, sequenceid=285, filesize=12.0 K 2024-11-12T14:18:49,404 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/52b64e4c174a45e38dc3b6a9f77ffb1e as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/52b64e4c174a45e38dc3b6a9f77ffb1e 2024-11-12T14:18:49,416 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:49,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421189412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:49,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,417 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:49,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:49,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421189412, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:49,417 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/52b64e4c174a45e38dc3b6a9f77ffb1e, entries=150, sequenceid=285, filesize=12.0 K 2024-11-12T14:18:49,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,418 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:49,421 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for 33074d5f6b8c90fb791f1838297603c6 in 312ms, sequenceid=285, compaction requested=false 2024-11-12T14:18:49,421 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:49,431 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T14:18:49,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:18:49,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:49,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:18:49,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:49,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:18:49,431 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:49,453 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/26481f14c7b046a1b6f6ac9a68c4ae14 is 50, key is test_row_0/A:col10/1731421129418/Put/seqid=0 2024-11-12T14:18:49,497 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742305_1481 (size=12301) 2024-11-12T14:18:49,526 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-12T14:18:49,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:49,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:49,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:49,527 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:49,527 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:49,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:49,548 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38622 deadline: 1731421189537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:49,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,550 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:49,557 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1731421189543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:49,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,557 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,558 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:49,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:49,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421189630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:49,641 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:49,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421189630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:49,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:49,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:49,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38622 deadline: 1731421189659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:49,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:49,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,661 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1731421189660, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:49,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:49,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:49,681 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-12T14:18:49,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:49,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:49,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:49,682 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:49,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:49,844 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-12T14:18:49,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:49,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:49,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:49,848 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:49,848 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:49,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-12T14:18:49,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:49,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38622 deadline: 1731421189870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:49,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:49,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1731421189870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:49,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,887 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:49,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/26481f14c7b046a1b6f6ac9a68c4ae14 2024-11-12T14:18:49,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/b7f4aaa3ed4043a8b0d87326126a10be is 50, key is test_row_0/B:col10/1731421129418/Put/seqid=0 2024-11-12T14:18:49,962 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:49,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421189949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:49,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,962 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:49,963 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:49,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421189950, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:49,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:49,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:49,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742306_1482 (size=12301) 2024-11-12T14:18:50,006 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-12T14:18:50,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:50,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:50,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:50,006 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:50,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:50,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:50,172 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-12T14:18:50,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:50,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:50,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:50,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:50,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:50,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:50,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:50,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1731421190191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:50,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:50,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:50,198 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:50,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:50,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38622 deadline: 1731421190200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:50,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:50,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:50,214 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:50,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:50,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421190300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:50,310 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_1' in 33074d5f6b8c90fb791f1838297603c6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:18:50,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:50,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:50,310 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:50,329 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-12T14:18:50,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:50,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:50,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:50,329 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:50,329 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:50,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:50,335 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-12T14:18:50,335 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-12T14:18:50,390 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/b7f4aaa3ed4043a8b0d87326126a10be 2024-11-12T14:18:50,418 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/b55b0d27f1bf4c6eac5110d7e20290b8 is 50, key is test_row_0/C:col10/1731421129418/Put/seqid=0 2024-11-12T14:18:50,464 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742307_1483 (size=12301) 2024-11-12T14:18:50,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:50,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421190471, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:50,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:50,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:50,488 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:50,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-12T14:18:50,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:50,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:50,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:50,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:50,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:50,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:50,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:50,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421190472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:50,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:50,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:50,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:50,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-12T14:18:50,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:50,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:50,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:50,649 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:50,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:50,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:50,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:50,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1731421190710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:50,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:50,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:50,729 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:50,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:50,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38622 deadline: 1731421190720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:50,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:50,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:50,732 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:50,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-12T14:18:50,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:50,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:50,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:50,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] handler.RSProcedureHandler(58): pid=116 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:50,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=116 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:50,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=116 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:50,867 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=306 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/b55b0d27f1bf4c6eac5110d7e20290b8 2024-11-12T14:18:50,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-12T14:18:50,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/26481f14c7b046a1b6f6ac9a68c4ae14 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/26481f14c7b046a1b6f6ac9a68c4ae14 2024-11-12T14:18:50,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/26481f14c7b046a1b6f6ac9a68c4ae14, entries=150, sequenceid=306, filesize=12.0 K 2024-11-12T14:18:50,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/b7f4aaa3ed4043a8b0d87326126a10be as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/b7f4aaa3ed4043a8b0d87326126a10be 2024-11-12T14:18:50,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/b7f4aaa3ed4043a8b0d87326126a10be, entries=150, sequenceid=306, filesize=12.0 K 2024-11-12T14:18:50,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/b55b0d27f1bf4c6eac5110d7e20290b8 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/b55b0d27f1bf4c6eac5110d7e20290b8 2024-11-12T14:18:50,901 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/b55b0d27f1bf4c6eac5110d7e20290b8, entries=150, sequenceid=306, filesize=12.0 K 2024-11-12T14:18:50,906 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 33074d5f6b8c90fb791f1838297603c6 in 1476ms, sequenceid=306, compaction requested=true 2024-11-12T14:18:50,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:50,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:18:50,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:50,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:18:50,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T14:18:50,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:18:50,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-12T14:18:50,907 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:50,908 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:50,913 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39889 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:50,913 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/A is initiating minor compaction (all files) 2024-11-12T14:18:50,913 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/A in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:50,913 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/96b7b8eae907406c9e117f29a8d82dea, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/5ef3858cda4c4a9891362161cc9146f2, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/26481f14c7b046a1b6f6ac9a68c4ae14] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=39.0 K 2024-11-12T14:18:50,914 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96b7b8eae907406c9e117f29a8d82dea, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731421128328 2024-11-12T14:18:50,914 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5ef3858cda4c4a9891362161cc9146f2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731421128538 2024-11-12T14:18:50,914 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 26481f14c7b046a1b6f6ac9a68c4ae14, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731421129272 2024-11-12T14:18:50,924 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:50,924 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/C is initiating minor compaction (all files) 2024-11-12T14:18:50,924 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/C in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:50,925 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/42f383bccb12469ea531d35295dd3adf, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/52b64e4c174a45e38dc3b6a9f77ffb1e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/b55b0d27f1bf4c6eac5110d7e20290b8] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=36.6 K 2024-11-12T14:18:50,928 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 42f383bccb12469ea531d35295dd3adf, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731421128328 2024-11-12T14:18:50,929 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 52b64e4c174a45e38dc3b6a9f77ffb1e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731421128538 2024-11-12T14:18:50,929 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting b55b0d27f1bf4c6eac5110d7e20290b8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731421129272 2024-11-12T14:18:50,939 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#A#compaction#405 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:50,939 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/71c81f2c17c14d7885c26906cc7628c0 is 50, key is test_row_0/A:col10/1731421129418/Put/seqid=0 2024-11-12T14:18:50,964 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=116 2024-11-12T14:18:50,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:50,964 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T14:18:50,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:18:50,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:50,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:18:50,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:50,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:18:50,964 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:50,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742308_1484 (size=12949) 2024-11-12T14:18:50,985 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#C#compaction#406 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:50,985 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/f9aafe01f08e41acac9ac8d114becf2a is 50, key is test_row_0/C:col10/1731421129418/Put/seqid=0 2024-11-12T14:18:51,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/40542fd13bb040938caee14c2bb5be69 is 50, key is test_row_0/A:col10/1731421129475/Put/seqid=0 2024-11-12T14:18:51,034 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742309_1485 (size=12949) 2024-11-12T14:18:51,077 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742310_1486 (size=12301) 2024-11-12T14:18:51,387 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/71c81f2c17c14d7885c26906cc7628c0 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/71c81f2c17c14d7885c26906cc7628c0 2024-11-12T14:18:51,407 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/A of 33074d5f6b8c90fb791f1838297603c6 into 71c81f2c17c14d7885c26906cc7628c0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:51,408 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:51,408 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/A, priority=13, startTime=1731421130907; duration=0sec 2024-11-12T14:18:51,408 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:51,408 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:A 2024-11-12T14:18:51,408 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:51,409 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37449 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:51,409 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/B is initiating minor compaction (all files) 2024-11-12T14:18:51,409 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/B in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:51,409 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/2a4958c66c1f42a4a0c84381b9ab9af9, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/cd6d782800df458ca7c6ad5f86fbf4d8, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/b7f4aaa3ed4043a8b0d87326126a10be] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=36.6 K 2024-11-12T14:18:51,409 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a4958c66c1f42a4a0c84381b9ab9af9, keycount=150, bloomtype=ROW, size=12.5 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731421128328 2024-11-12T14:18:51,409 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd6d782800df458ca7c6ad5f86fbf4d8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731421128538 2024-11-12T14:18:51,410 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7f4aaa3ed4043a8b0d87326126a10be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731421129272 2024-11-12T14:18:51,424 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#B#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:51,424 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/984a8404d36b47b69bae5ba17f92cde6 is 50, key is test_row_0/B:col10/1731421129418/Put/seqid=0 2024-11-12T14:18:51,443 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/f9aafe01f08e41acac9ac8d114becf2a as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/f9aafe01f08e41acac9ac8d114becf2a 2024-11-12T14:18:51,450 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742311_1487 (size=12949) 2024-11-12T14:18:51,451 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/C of 33074d5f6b8c90fb791f1838297603c6 into f9aafe01f08e41acac9ac8d114becf2a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:51,451 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:51,451 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/C, priority=13, startTime=1731421130907; duration=0sec 2024-11-12T14:18:51,451 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:51,451 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:C 2024-11-12T14:18:51,478 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/984a8404d36b47b69bae5ba17f92cde6 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/984a8404d36b47b69bae5ba17f92cde6 2024-11-12T14:18:51,478 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/40542fd13bb040938caee14c2bb5be69 2024-11-12T14:18:51,538 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:51,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:51,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/fedf0ceb3f4547a28a415db80ff389fc is 50, key is test_row_0/B:col10/1731421129475/Put/seqid=0 2024-11-12T14:18:51,558 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/B of 33074d5f6b8c90fb791f1838297603c6 into 984a8404d36b47b69bae5ba17f92cde6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:51,559 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:51,559 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/B, priority=13, startTime=1731421130907; duration=0sec 2024-11-12T14:18:51,559 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:51,559 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:B 2024-11-12T14:18:51,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742312_1488 (size=12301) 2024-11-12T14:18:51,586 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/fedf0ceb3f4547a28a415db80ff389fc 2024-11-12T14:18:51,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/268e2f5a17954459a5325ebc83fc1c8f is 50, key is test_row_0/C:col10/1731421129475/Put/seqid=0 2024-11-12T14:18:51,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742313_1489 (size=12301) 2024-11-12T14:18:51,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:51,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421191662, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:51,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:51,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:51,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:51,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421191669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:51,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:51,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:51,675 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:51,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:51,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1731421191741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:51,752 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:51,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38622 deadline: 1731421191750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:51,754 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_2' in 33074d5f6b8c90fb791f1838297603c6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:18:51,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:51,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:51,754 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:51,760 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_2' in 33074d5f6b8c90fb791f1838297603c6 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:18:51,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:51,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:51,761 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:51,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421191779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:51,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:51,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:51,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:51,786 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421191780, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:51,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:51,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:51,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:51,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:51,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421191989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:51,991 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:51,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421191989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:51,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:51,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:51,992 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:51,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:51,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:51,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:52,008 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=320 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/268e2f5a17954459a5325ebc83fc1c8f 2024-11-12T14:18:52,016 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/40542fd13bb040938caee14c2bb5be69 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/40542fd13bb040938caee14c2bb5be69 2024-11-12T14:18:52,020 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/40542fd13bb040938caee14c2bb5be69, entries=150, sequenceid=320, filesize=12.0 K 2024-11-12T14:18:52,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/fedf0ceb3f4547a28a415db80ff389fc as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/fedf0ceb3f4547a28a415db80ff389fc 2024-11-12T14:18:52,028 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/fedf0ceb3f4547a28a415db80ff389fc, entries=150, sequenceid=320, filesize=12.0 K 2024-11-12T14:18:52,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/268e2f5a17954459a5325ebc83fc1c8f as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/268e2f5a17954459a5325ebc83fc1c8f 2024-11-12T14:18:52,035 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/268e2f5a17954459a5325ebc83fc1c8f, entries=150, sequenceid=320, filesize=12.0 K 2024-11-12T14:18:52,036 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 33074d5f6b8c90fb791f1838297603c6 in 1071ms, sequenceid=320, compaction requested=false 2024-11-12T14:18:52,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.HRegion(2603): Flush status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:52,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:52,036 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=116}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=116 2024-11-12T14:18:52,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=116 2024-11-12T14:18:52,040 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-12T14:18:52,040 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3120 sec 2024-11-12T14:18:52,050 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=115, table=TestAcidGuarantees in 3.3200 sec 2024-11-12T14:18:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:52,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T14:18:52,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:18:52,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:52,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:18:52,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:52,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:18:52,305 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:52,311 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/82affaad25c04e39846fa8cb02bfe232 is 50, key is test_row_0/A:col10/1731421132303/Put/seqid=0 2024-11-12T14:18:52,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742314_1490 (size=17181) 2024-11-12T14:18:52,327 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/82affaad25c04e39846fa8cb02bfe232 2024-11-12T14:18:52,350 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/593ab6e5137f4ec7adaf9f254baa99bf is 50, key is test_row_0/B:col10/1731421132303/Put/seqid=0 2024-11-12T14:18:52,366 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421192352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:52,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:52,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:52,367 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:52,366 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:52,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421192351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421192353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:52,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:52,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:52,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:52,368 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:52,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:52,368 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:52,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742315_1491 (size=12301) 2024-11-12T14:18:52,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421192469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:52,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:52,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:52,478 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421192469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:52,479 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:52,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421192479, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:52,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:52,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:52,484 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:52,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:52,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421192689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:52,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:52,696 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:52,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:52,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:52,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421192689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:52,697 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:52,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:52,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:52,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421192689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:52,697 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:52,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:52,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:52,697 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:52,780 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/593ab6e5137f4ec7adaf9f254baa99bf 2024-11-12T14:18:52,834 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/bbf492a2b2454b93bd5149cc3ae55523 is 50, key is test_row_0/C:col10/1731421132303/Put/seqid=0 2024-11-12T14:18:52,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742316_1492 (size=12301) 2024-11-12T14:18:52,860 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/bbf492a2b2454b93bd5149cc3ae55523 2024-11-12T14:18:52,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-12T14:18:52,880 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:18:52,881 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:18:52,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees 2024-11-12T14:18:52,884 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:18:52,885 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=117, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:18:52,885 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:18:52,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-12T14:18:52,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/82affaad25c04e39846fa8cb02bfe232 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/82affaad25c04e39846fa8cb02bfe232 2024-11-12T14:18:52,903 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/82affaad25c04e39846fa8cb02bfe232, entries=250, sequenceid=343, filesize=16.8 K 2024-11-12T14:18:52,904 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/593ab6e5137f4ec7adaf9f254baa99bf as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/593ab6e5137f4ec7adaf9f254baa99bf 2024-11-12T14:18:52,911 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/593ab6e5137f4ec7adaf9f254baa99bf, entries=150, sequenceid=343, filesize=12.0 K 2024-11-12T14:18:52,912 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/bbf492a2b2454b93bd5149cc3ae55523 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/bbf492a2b2454b93bd5149cc3ae55523 2024-11-12T14:18:52,917 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/bbf492a2b2454b93bd5149cc3ae55523, entries=150, sequenceid=343, filesize=12.0 K 2024-11-12T14:18:52,918 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 33074d5f6b8c90fb791f1838297603c6 in 615ms, sequenceid=343, compaction requested=true 2024-11-12T14:18:52,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:52,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:18:52,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:52,918 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:18:52,918 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:52,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:52,919 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:52,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:18:52,919 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:52,923 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42431 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:52,923 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/A is initiating minor compaction (all files) 2024-11-12T14:18:52,923 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/A in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:52,923 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/71c81f2c17c14d7885c26906cc7628c0, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/40542fd13bb040938caee14c2bb5be69, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/82affaad25c04e39846fa8cb02bfe232] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=41.4 K 2024-11-12T14:18:52,925 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:52,925 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/B is initiating minor compaction (all files) 2024-11-12T14:18:52,925 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/B in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:52,925 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/984a8404d36b47b69bae5ba17f92cde6, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/fedf0ceb3f4547a28a415db80ff389fc, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/593ab6e5137f4ec7adaf9f254baa99bf] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=36.7 K 2024-11-12T14:18:52,925 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71c81f2c17c14d7885c26906cc7628c0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731421129272 2024-11-12T14:18:52,925 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 984a8404d36b47b69bae5ba17f92cde6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731421129272 2024-11-12T14:18:52,925 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 40542fd13bb040938caee14c2bb5be69, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731421129475 2024-11-12T14:18:52,926 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting fedf0ceb3f4547a28a415db80ff389fc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731421129475 2024-11-12T14:18:52,926 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82affaad25c04e39846fa8cb02bfe232, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731421131647 2024-11-12T14:18:52,926 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 593ab6e5137f4ec7adaf9f254baa99bf, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731421131660 2024-11-12T14:18:52,933 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#A#compaction#414 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:52,933 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/4f91e0838e5246fd81631879b126bf99 is 50, key is test_row_0/A:col10/1731421132303/Put/seqid=0 2024-11-12T14:18:52,938 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#B#compaction#415 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:52,939 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/8cf6f8e15c82429daeeb603dfa64a055 is 50, key is test_row_0/B:col10/1731421132303/Put/seqid=0 2024-11-12T14:18:52,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742317_1493 (size=13051) 2024-11-12T14:18:52,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742318_1494 (size=13051) 2024-11-12T14:18:52,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-12T14:18:52,992 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/4f91e0838e5246fd81631879b126bf99 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/4f91e0838e5246fd81631879b126bf99 2024-11-12T14:18:52,998 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/8cf6f8e15c82429daeeb603dfa64a055 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/8cf6f8e15c82429daeeb603dfa64a055 2024-11-12T14:18:52,999 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/A of 33074d5f6b8c90fb791f1838297603c6 into 4f91e0838e5246fd81631879b126bf99(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:52,999 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:52,999 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/A, priority=13, startTime=1731421132918; duration=0sec 2024-11-12T14:18:52,999 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:52,999 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:A 2024-11-12T14:18:52,999 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:53,000 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:53,000 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/C is initiating minor compaction (all files) 2024-11-12T14:18:53,000 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/C in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:53,000 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/f9aafe01f08e41acac9ac8d114becf2a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/268e2f5a17954459a5325ebc83fc1c8f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/bbf492a2b2454b93bd5149cc3ae55523] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=36.7 K 2024-11-12T14:18:53,001 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting f9aafe01f08e41acac9ac8d114becf2a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=306, earliestPutTs=1731421129272 2024-11-12T14:18:53,004 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 268e2f5a17954459a5325ebc83fc1c8f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=320, earliestPutTs=1731421129475 2024-11-12T14:18:53,005 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting bbf492a2b2454b93bd5149cc3ae55523, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731421131660 2024-11-12T14:18:53,007 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/B of 33074d5f6b8c90fb791f1838297603c6 into 8cf6f8e15c82429daeeb603dfa64a055(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:53,008 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:53,008 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/B, priority=13, startTime=1731421132918; duration=0sec 2024-11-12T14:18:53,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:53,008 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:53,008 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:B 2024-11-12T14:18:53,012 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T14:18:53,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:18:53,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:53,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:18:53,012 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:53,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:18:53,013 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:53,014 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#C#compaction#416 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:53,014 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/441aac4aed7a4f2b973e1f8ce1734fb4 is 50, key is test_row_0/C:col10/1731421132303/Put/seqid=0 2024-11-12T14:18:53,021 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/62a3011546514d0ba3762e89b0811c09 is 50, key is test_row_0/A:col10/1731421133006/Put/seqid=0 2024-11-12T14:18:53,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742319_1495 (size=13051) 2024-11-12T14:18:53,032 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/441aac4aed7a4f2b973e1f8ce1734fb4 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/441aac4aed7a4f2b973e1f8ce1734fb4 2024-11-12T14:18:53,036 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/C of 33074d5f6b8c90fb791f1838297603c6 into 441aac4aed7a4f2b973e1f8ce1734fb4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:53,036 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:53,036 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/C, priority=13, startTime=1731421132919; duration=0sec 2024-11-12T14:18:53,036 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:53,036 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:C 2024-11-12T14:18:53,041 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-12T14:18:53,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:53,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:53,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:53,041 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:53,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:53,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:53,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742320_1496 (size=12301) 2024-11-12T14:18:53,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/62a3011546514d0ba3762e89b0811c09 2024-11-12T14:18:53,052 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/0c3e6de1bffe41908197a5d598d86646 is 50, key is test_row_0/B:col10/1731421133006/Put/seqid=0 2024-11-12T14:18:53,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742321_1497 (size=12301) 2024-11-12T14:18:53,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:53,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:53,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421193059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:53,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421193058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:53,072 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:53,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421193069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:53,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:53,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:53,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:53,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:53,074 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:53,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:53,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:53,074 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:53,074 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:53,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:53,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421193179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:53,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:53,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:53,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421193180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:53,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:53,186 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:53,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:53,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:53,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:53,186 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:53,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421193180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:53,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:53,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:53,187 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:53,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-12T14:18:53,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:53,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:53,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:53,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:53,194 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:53,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:53,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-12T14:18:53,346 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-12T14:18:53,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:53,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:53,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:53,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:53,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:53,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:53,397 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:53,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421193389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:53,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:53,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421193389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:53,397 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:53,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421193389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:53,398 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:53,459 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/0c3e6de1bffe41908197a5d598d86646 2024-11-12T14:18:53,466 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/f8b345bc14494f27930872337f7c801b is 50, key is test_row_0/C:col10/1731421133006/Put/seqid=0 2024-11-12T14:18:53,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742322_1498 (size=12301) 2024-11-12T14:18:53,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-12T14:18:53,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:53,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:53,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:53,499 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:53,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:53,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:53,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-12T14:18:53,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-12T14:18:53,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:53,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:53,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:53,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:53,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:53,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:53,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:53,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421193710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:53,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:53,716 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:53,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:53,716 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:53,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421193710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:53,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:53,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:53,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:53,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421193710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:53,717 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:53,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:53,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:53,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:53,778 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:53,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1731421193769, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:53,778 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_2' in 33074d5f6b8c90fb791f1838297603c6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:18:53,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:53,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:53,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:53,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:53,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38622 deadline: 1731421193779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:53,787 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_2' in 33074d5f6b8c90fb791f1838297603c6 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4250 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:18:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:53,787 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:53,804 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-12T14:18:53,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:53,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:53,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:53,804 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] handler.RSProcedureHandler(58): pid=118 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:53,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=118 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:53,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=118 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:53,806 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=45591 {}] assignment.AssignmentManager(1535): Unable to acquire lock for regionNode state=OPEN, location=2b6d221c5cde,42925,1731420990822, table=TestAcidGuarantees, region=33074d5f6b8c90fb791f1838297603c6. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-12T14:18:53,870 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=359 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/f8b345bc14494f27930872337f7c801b 2024-11-12T14:18:53,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/62a3011546514d0ba3762e89b0811c09 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/62a3011546514d0ba3762e89b0811c09 2024-11-12T14:18:53,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/62a3011546514d0ba3762e89b0811c09, entries=150, sequenceid=359, filesize=12.0 K 2024-11-12T14:18:53,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/0c3e6de1bffe41908197a5d598d86646 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/0c3e6de1bffe41908197a5d598d86646 2024-11-12T14:18:53,888 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/0c3e6de1bffe41908197a5d598d86646, entries=150, sequenceid=359, filesize=12.0 K 2024-11-12T14:18:53,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/f8b345bc14494f27930872337f7c801b as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/f8b345bc14494f27930872337f7c801b 2024-11-12T14:18:53,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/f8b345bc14494f27930872337f7c801b, entries=150, sequenceid=359, filesize=12.0 K 2024-11-12T14:18:53,893 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for 33074d5f6b8c90fb791f1838297603c6 in 881ms, sequenceid=359, compaction requested=false 2024-11-12T14:18:53,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:53,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=118 2024-11-12T14:18:53,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:53,957 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T14:18:53,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:18:53,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:53,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:18:53,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:53,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:18:53,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:53,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/32a110ccba734597ad2622128f59d786 is 50, key is test_row_0/A:col10/1731421133055/Put/seqid=0 2024-11-12T14:18:53,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742323_1499 (size=12301) 2024-11-12T14:18:53,971 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/32a110ccba734597ad2622128f59d786 2024-11-12T14:18:53,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/7396fcffc53046a4bd4bb72e097f0ad5 is 50, key is test_row_0/B:col10/1731421133055/Put/seqid=0 2024-11-12T14:18:54,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742324_1500 (size=12301) 2024-11-12T14:18:54,006 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/7396fcffc53046a4bd4bb72e097f0ad5 2024-11-12T14:18:54,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-12T14:18:54,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/7c18747ed60f40f188b8f753c4eef919 is 50, key is test_row_0/C:col10/1731421133055/Put/seqid=0 2024-11-12T14:18:54,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742325_1501 (size=12301) 2024-11-12T14:18:54,042 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=378 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/7c18747ed60f40f188b8f753c4eef919 2024-11-12T14:18:54,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/32a110ccba734597ad2622128f59d786 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/32a110ccba734597ad2622128f59d786 2024-11-12T14:18:54,058 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/32a110ccba734597ad2622128f59d786, entries=150, sequenceid=378, filesize=12.0 K 2024-11-12T14:18:54,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/7396fcffc53046a4bd4bb72e097f0ad5 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/7396fcffc53046a4bd4bb72e097f0ad5 2024-11-12T14:18:54,064 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/7396fcffc53046a4bd4bb72e097f0ad5, entries=150, sequenceid=378, filesize=12.0 K 2024-11-12T14:18:54,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/7c18747ed60f40f188b8f753c4eef919 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/7c18747ed60f40f188b8f753c4eef919 2024-11-12T14:18:54,069 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/7c18747ed60f40f188b8f753c4eef919, entries=150, sequenceid=378, filesize=12.0 K 2024-11-12T14:18:54,070 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=0 B/0 for 33074d5f6b8c90fb791f1838297603c6 in 113ms, sequenceid=378, compaction requested=true 2024-11-12T14:18:54,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.HRegion(2603): Flush status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:54,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:54,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=118}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=118 2024-11-12T14:18:54,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=118 2024-11-12T14:18:54,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-12T14:18:54,073 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1860 sec 2024-11-12T14:18:54,075 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=117, table=TestAcidGuarantees in 1.1930 sec 2024-11-12T14:18:54,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:54,232 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T14:18:54,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:18:54,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:54,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:18:54,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:54,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:18:54,232 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:54,237 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/3123d4c7670d495dab3c8f7314afbe46 is 50, key is test_row_0/A:col10/1731421134229/Put/seqid=0 2024-11-12T14:18:54,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742326_1502 (size=17181) 2024-11-12T14:18:54,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731421134285, max timestamp=1731421134305Num uniques 859; 2024-11-12T14:18:54,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T14:18:54,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731421134285, max timestamp=1731421134305Num uniques 859; 2024-11-12T14:18:54,308 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T14:18:54,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-1 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=259, cellSize=11855, totalHeapSize=27176, min timestamp=1731421134285, max timestamp=1731421134305Num uniques 859; 2024-11-12T14:18:54,310 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-1 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=11855, suffix heap size=117064,new segment heap size=27176 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=259 2024-11-12T14:18:54,364 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:54,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421194352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:54,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:54,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:54,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:54,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:54,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421194363, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:54,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:54,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:54,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:54,378 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:54,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421194364, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:54,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:54,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:54,379 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:54,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:54,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421194469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:54,479 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:54,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421194470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:54,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:54,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:54,481 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:54,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:54,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:54,482 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:54,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:54,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421194489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:54,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:54,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:54,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:54,653 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/3123d4c7670d495dab3c8f7314afbe46 2024-11-12T14:18:54,659 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/7a2bf67f29304ab69c5519bd104cc9d1 is 50, key is test_row_0/B:col10/1731421134229/Put/seqid=0 2024-11-12T14:18:54,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742327_1503 (size=12301) 2024-11-12T14:18:54,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:54,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421194689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:54,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:54,700 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:54,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:54,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:54,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421194689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:54,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:54,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:54,700 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:54,705 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:54,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421194699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:54,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:54,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:54,705 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:55,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:55,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421195009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:55,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421195009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:55,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:55,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421195009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:55,021 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:55,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:55,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:55,026 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:55,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-12T14:18:55,029 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:18:55,029 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:18:55,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-12T14:18:55,031 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:18:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-12T14:18:55,031 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:18:55,031 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:18:55,064 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/7a2bf67f29304ab69c5519bd104cc9d1 2024-11-12T14:18:55,089 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/36eb1aa6901f49e6b60863f5bb17deef is 50, key is test_row_0/C:col10/1731421134229/Put/seqid=0 2024-11-12T14:18:55,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742328_1504 (size=12301) 2024-11-12T14:18:55,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-12T14:18:55,150 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/36eb1aa6901f49e6b60863f5bb17deef 2024-11-12T14:18:55,168 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/3123d4c7670d495dab3c8f7314afbe46 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/3123d4c7670d495dab3c8f7314afbe46 2024-11-12T14:18:55,171 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/3123d4c7670d495dab3c8f7314afbe46, entries=250, sequenceid=388, filesize=16.8 K 2024-11-12T14:18:55,172 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/7a2bf67f29304ab69c5519bd104cc9d1 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/7a2bf67f29304ab69c5519bd104cc9d1 2024-11-12T14:18:55,175 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/7a2bf67f29304ab69c5519bd104cc9d1, entries=150, sequenceid=388, filesize=12.0 K 2024-11-12T14:18:55,176 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/36eb1aa6901f49e6b60863f5bb17deef as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/36eb1aa6901f49e6b60863f5bb17deef 2024-11-12T14:18:55,180 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/36eb1aa6901f49e6b60863f5bb17deef, entries=150, sequenceid=388, filesize=12.0 K 2024-11-12T14:18:55,181 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=134.18 KB/137400 for 33074d5f6b8c90fb791f1838297603c6 in 949ms, sequenceid=388, compaction requested=true 2024-11-12T14:18:55,181 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:55,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:18:55,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:55,181 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:18:55,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:18:55,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:55,181 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:18:55,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:18:55,181 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:55,182 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54834 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:18:55,182 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:18:55,183 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/A is initiating minor compaction (all files) 2024-11-12T14:18:55,183 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/B is initiating minor compaction (all files) 2024-11-12T14:18:55,183 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/A in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:55,183 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/B in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:55,183 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/4f91e0838e5246fd81631879b126bf99, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/62a3011546514d0ba3762e89b0811c09, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/32a110ccba734597ad2622128f59d786, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/3123d4c7670d495dab3c8f7314afbe46] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=53.5 K 2024-11-12T14:18:55,183 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/8cf6f8e15c82429daeeb603dfa64a055, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/0c3e6de1bffe41908197a5d598d86646, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/7396fcffc53046a4bd4bb72e097f0ad5, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/7a2bf67f29304ab69c5519bd104cc9d1] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=48.8 K 2024-11-12T14:18:55,183 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f91e0838e5246fd81631879b126bf99, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731421131660 2024-11-12T14:18:55,183 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 8cf6f8e15c82429daeeb603dfa64a055, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731421131660 2024-11-12T14:18:55,183 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62a3011546514d0ba3762e89b0811c09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731421132333 2024-11-12T14:18:55,183 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c3e6de1bffe41908197a5d598d86646, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731421132333 2024-11-12T14:18:55,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-12T14:18:55,184 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 32a110ccba734597ad2622128f59d786, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731421133055 2024-11-12T14:18:55,184 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 7396fcffc53046a4bd4bb72e097f0ad5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731421133055 2024-11-12T14:18:55,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:55,184 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=134.18 KB heapSize=375.61 KB 2024-11-12T14:18:55,184 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3123d4c7670d495dab3c8f7314afbe46, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731421134221 2024-11-12T14:18:55,184 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a2bf67f29304ab69c5519bd104cc9d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731421134221 2024-11-12T14:18:55,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:18:55,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:18:55,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:18:55,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:18:55,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:18:55,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:18:55,195 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/3de296d5ebb34c499dee98daea87faa3 is 50, key is test_row_0/A:col10/1731421134363/Put/seqid=0 2024-11-12T14:18:55,203 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#A#compaction#427 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:55,204 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/63270c58329e4ad092e1a1c65eb022cd is 50, key is test_row_0/A:col10/1731421134229/Put/seqid=0 2024-11-12T14:18:55,208 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#B#compaction#428 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:55,208 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/87de762ab69d42b29e647eb544c53ce2 is 50, key is test_row_0/B:col10/1731421134229/Put/seqid=0 2024-11-12T14:18:55,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742330_1506 (size=13187) 2024-11-12T14:18:55,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742329_1505 (size=12301) 2024-11-12T14:18:55,294 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742331_1507 (size=13187) 2024-11-12T14:18:55,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-12T14:18:55,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:55,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:55,578 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:55,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421195566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:55,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:55,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:55,578 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:55,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:55,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421195580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:55,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:55,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421195580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:55,599 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:55,655 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/3de296d5ebb34c499dee98daea87faa3 2024-11-12T14:18:55,665 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/63270c58329e4ad092e1a1c65eb022cd as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/63270c58329e4ad092e1a1c65eb022cd 2024-11-12T14:18:55,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-12T14:18:55,677 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/A of 33074d5f6b8c90fb791f1838297603c6 into 63270c58329e4ad092e1a1c65eb022cd(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:55,677 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:55,677 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/A, priority=12, startTime=1731421135181; duration=0sec 2024-11-12T14:18:55,678 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:55,678 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:A 2024-11-12T14:18:55,678 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:18:55,679 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:18:55,679 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/C is initiating minor compaction (all files) 2024-11-12T14:18:55,679 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/C in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:55,679 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/441aac4aed7a4f2b973e1f8ce1734fb4, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/f8b345bc14494f27930872337f7c801b, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/7c18747ed60f40f188b8f753c4eef919, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/36eb1aa6901f49e6b60863f5bb17deef] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=48.8 K 2024-11-12T14:18:55,680 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 441aac4aed7a4f2b973e1f8ce1734fb4, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1731421131660 2024-11-12T14:18:55,680 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8b345bc14494f27930872337f7c801b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=359, earliestPutTs=1731421132333 2024-11-12T14:18:55,680 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c18747ed60f40f188b8f753c4eef919, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=378, earliestPutTs=1731421133055 2024-11-12T14:18:55,681 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36eb1aa6901f49e6b60863f5bb17deef, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731421134221 2024-11-12T14:18:55,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/986b81e336514f3da51f19a76e08749f is 50, key is test_row_0/B:col10/1731421134363/Put/seqid=0 2024-11-12T14:18:55,697 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/87de762ab69d42b29e647eb544c53ce2 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/87de762ab69d42b29e647eb544c53ce2 2024-11-12T14:18:55,698 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:55,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421195690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:55,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:55,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:55,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:55,703 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#C#compaction#430 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:55,704 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/254fa06fb4294e2a940e2382acbc7ee8 is 50, key is test_row_0/C:col10/1731421134229/Put/seqid=0 2024-11-12T14:18:55,704 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/B of 33074d5f6b8c90fb791f1838297603c6 into 87de762ab69d42b29e647eb544c53ce2(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:55,704 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:55,704 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/B, priority=12, startTime=1731421135181; duration=0sec 2024-11-12T14:18:55,705 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:55,705 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:B 2024-11-12T14:18:55,716 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:55,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421195710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:55,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:55,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:55,717 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:55,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742332_1508 (size=12301) 2024-11-12T14:18:55,731 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421195711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:55,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:55,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:55,731 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:55,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742333_1509 (size=13187) 2024-11-12T14:18:55,764 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/254fa06fb4294e2a940e2382acbc7ee8 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/254fa06fb4294e2a940e2382acbc7ee8 2024-11-12T14:18:55,776 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/C of 33074d5f6b8c90fb791f1838297603c6 into 254fa06fb4294e2a940e2382acbc7ee8(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:55,776 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:55,776 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/C, priority=12, startTime=1731421135181; duration=0sec 2024-11-12T14:18:55,776 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:55,776 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:C 2024-11-12T14:18:55,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:55,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421195913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:55,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:55,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:55,922 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:55,937 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:55,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421195929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:55,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:55,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:55,937 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:55,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:55,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421195939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:55,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:55,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:55,946 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:56,121 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/986b81e336514f3da51f19a76e08749f 2024-11-12T14:18:56,130 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/c991fe6ce335435ca9a6419d1b8d5d35 is 50, key is test_row_0/C:col10/1731421134363/Put/seqid=0 2024-11-12T14:18:56,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742334_1510 (size=12301) 2024-11-12T14:18:56,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-12T14:18:56,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421196229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:56,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:56,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:56,241 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:56,245 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:56,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421196239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:56,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:56,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:56,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:56,271 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:56,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421196260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:56,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:56,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:56,271 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:56,535 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=423 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/c991fe6ce335435ca9a6419d1b8d5d35 2024-11-12T14:18:56,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/3de296d5ebb34c499dee98daea87faa3 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/3de296d5ebb34c499dee98daea87faa3 2024-11-12T14:18:56,542 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/3de296d5ebb34c499dee98daea87faa3, entries=150, sequenceid=423, filesize=12.0 K 2024-11-12T14:18:56,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/986b81e336514f3da51f19a76e08749f as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/986b81e336514f3da51f19a76e08749f 2024-11-12T14:18:56,546 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/986b81e336514f3da51f19a76e08749f, entries=150, sequenceid=423, filesize=12.0 K 2024-11-12T14:18:56,549 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/c991fe6ce335435ca9a6419d1b8d5d35 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/c991fe6ce335435ca9a6419d1b8d5d35 2024-11-12T14:18:56,558 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/c991fe6ce335435ca9a6419d1b8d5d35, entries=150, sequenceid=423, filesize=12.0 K 2024-11-12T14:18:56,559 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~375.56 KB/384576, currentSize=53.67 KB/54960 for 33074d5f6b8c90fb791f1838297603c6 in 1375ms, sequenceid=423, compaction requested=false 2024-11-12T14:18:56,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:56,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:56,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-12T14:18:56,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-12T14:18:56,565 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-12T14:18:56,565 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5320 sec 2024-11-12T14:18:56,566 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.5360 sec 2024-11-12T14:18:56,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:56,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T14:18:56,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:18:56,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:56,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:18:56,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:56,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:18:56,757 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:56,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/87006a1eed204fa9b9e91eab7fb90046 is 50, key is test_row_0/A:col10/1731421135543/Put/seqid=0 2024-11-12T14:18:56,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742335_1511 (size=14741) 2024-11-12T14:18:56,796 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/87006a1eed204fa9b9e91eab7fb90046 2024-11-12T14:18:56,807 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/528b03a311ee4cf5b19807bfe669b928 is 50, key is test_row_0/B:col10/1731421135543/Put/seqid=0 2024-11-12T14:18:56,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:56,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421196830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:56,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742336_1512 (size=12301) 2024-11-12T14:18:56,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:56,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421196833, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:56,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:56,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:56,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:56,841 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:56,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:56,841 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:56,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/528b03a311ee4cf5b19807bfe669b928 2024-11-12T14:18:56,847 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/7033efc722ce43b2aa60c9316ea82c83 is 50, key is test_row_0/C:col10/1731421135543/Put/seqid=0 2024-11-12T14:18:56,848 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:56,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421196840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:56,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:56,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:56,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:56,852 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742337_1513 (size=12301) 2024-11-12T14:18:56,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:56,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421196949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:56,949 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:56,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421196949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:56,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:56,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:56,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:56,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:56,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:56,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:56,962 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:56,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421196959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:56,963 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:56,964 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:57,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:57,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421197160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:57,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:57,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421197160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:57,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:57,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:57,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:57,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:57,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:57,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:57,176 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:57,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421197169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:57,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:57,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:57,177 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:57,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-12T14:18:57,189 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:18:57,190 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:18:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-12T14:18:57,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T14:18:57,192 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:18:57,192 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:18:57,192 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:18:57,257 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=438 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/7033efc722ce43b2aa60c9316ea82c83 2024-11-12T14:18:57,270 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/87006a1eed204fa9b9e91eab7fb90046 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/87006a1eed204fa9b9e91eab7fb90046 2024-11-12T14:18:57,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/87006a1eed204fa9b9e91eab7fb90046, entries=200, sequenceid=438, filesize=14.4 K 2024-11-12T14:18:57,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/528b03a311ee4cf5b19807bfe669b928 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/528b03a311ee4cf5b19807bfe669b928 2024-11-12T14:18:57,286 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/528b03a311ee4cf5b19807bfe669b928, entries=150, sequenceid=438, filesize=12.0 K 2024-11-12T14:18:57,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/7033efc722ce43b2aa60c9316ea82c83 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/7033efc722ce43b2aa60c9316ea82c83 2024-11-12T14:18:57,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/7033efc722ce43b2aa60c9316ea82c83, entries=150, sequenceid=438, filesize=12.0 K 2024-11-12T14:18:57,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for 33074d5f6b8c90fb791f1838297603c6 in 536ms, sequenceid=438, compaction requested=true 2024-11-12T14:18:57,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:57,294 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:57,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:18:57,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:57,294 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:57,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:18:57,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:57,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:18:57,298 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:57,299 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:57,299 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/A is initiating minor compaction (all files) 2024-11-12T14:18:57,299 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/A in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:57,299 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/63270c58329e4ad092e1a1c65eb022cd, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/3de296d5ebb34c499dee98daea87faa3, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/87006a1eed204fa9b9e91eab7fb90046] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=39.3 K 2024-11-12T14:18:57,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T14:18:57,300 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:57,300 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/B is initiating minor compaction (all files) 2024-11-12T14:18:57,300 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/B in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:57,300 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/87de762ab69d42b29e647eb544c53ce2, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/986b81e336514f3da51f19a76e08749f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/528b03a311ee4cf5b19807bfe669b928] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=36.9 K 2024-11-12T14:18:57,300 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63270c58329e4ad092e1a1c65eb022cd, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731421134221 2024-11-12T14:18:57,300 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 87de762ab69d42b29e647eb544c53ce2, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731421134221 2024-11-12T14:18:57,300 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3de296d5ebb34c499dee98daea87faa3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731421134344 2024-11-12T14:18:57,300 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 986b81e336514f3da51f19a76e08749f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731421134344 2024-11-12T14:18:57,301 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 528b03a311ee4cf5b19807bfe669b928, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731421135543 2024-11-12T14:18:57,301 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 87006a1eed204fa9b9e91eab7fb90046, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731421135543 2024-11-12T14:18:57,322 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#B#compaction#435 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:57,323 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/d0183cc522c24542a2b1e40c98c0a28f is 50, key is test_row_0/B:col10/1731421135543/Put/seqid=0 2024-11-12T14:18:57,334 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#A#compaction#436 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:57,335 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/7ad045ddd4df48929b3128b5697177c1 is 50, key is test_row_0/A:col10/1731421135543/Put/seqid=0 2024-11-12T14:18:57,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-12T14:18:57,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:57,348 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T14:18:57,348 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:18:57,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:57,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:18:57,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:57,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:18:57,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:57,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742338_1514 (size=13289) 2024-11-12T14:18:57,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/b70eaf8e714a4605b7b9d91707cac7eb is 50, key is test_row_0/A:col10/1731421136832/Put/seqid=0 2024-11-12T14:18:57,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742339_1515 (size=13289) 2024-11-12T14:18:57,426 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/7ad045ddd4df48929b3128b5697177c1 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/7ad045ddd4df48929b3128b5697177c1 2024-11-12T14:18:57,432 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/A of 33074d5f6b8c90fb791f1838297603c6 into 7ad045ddd4df48929b3128b5697177c1(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:57,433 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:57,433 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/A, priority=13, startTime=1731421137294; duration=0sec 2024-11-12T14:18:57,433 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:57,433 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:A 2024-11-12T14:18:57,433 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:57,434 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:57,435 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/C is initiating minor compaction (all files) 2024-11-12T14:18:57,435 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/C in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:57,435 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/254fa06fb4294e2a940e2382acbc7ee8, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/c991fe6ce335435ca9a6419d1b8d5d35, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/7033efc722ce43b2aa60c9316ea82c83] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=36.9 K 2024-11-12T14:18:57,435 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 254fa06fb4294e2a940e2382acbc7ee8, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1731421134221 2024-11-12T14:18:57,436 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting c991fe6ce335435ca9a6419d1b8d5d35, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=423, earliestPutTs=1731421134344 2024-11-12T14:18:57,436 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7033efc722ce43b2aa60c9316ea82c83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731421135543 2024-11-12T14:18:57,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742340_1516 (size=12301) 2024-11-12T14:18:57,442 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/b70eaf8e714a4605b7b9d91707cac7eb 2024-11-12T14:18:57,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/dbb40c0bdb1644148b606803bdef487c is 50, key is test_row_0/B:col10/1731421136832/Put/seqid=0 2024-11-12T14:18:57,467 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#C#compaction#439 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:57,467 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/8b3310ae425a489383b6ea8b15b72b33 is 50, key is test_row_0/C:col10/1731421135543/Put/seqid=0 2024-11-12T14:18:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:57,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:57,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742341_1517 (size=12301) 2024-11-12T14:18:57,490 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742342_1518 (size=13289) 2024-11-12T14:18:57,490 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/dbb40c0bdb1644148b606803bdef487c 2024-11-12T14:18:57,502 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/b4de65f5894f4f52b03b03dcbb1c0557 is 50, key is test_row_0/C:col10/1731421136832/Put/seqid=0 2024-11-12T14:18:57,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T14:18:57,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742343_1519 (size=12301) 2024-11-12T14:18:57,509 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=458 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/b4de65f5894f4f52b03b03dcbb1c0557 2024-11-12T14:18:57,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/b70eaf8e714a4605b7b9d91707cac7eb as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/b70eaf8e714a4605b7b9d91707cac7eb 2024-11-12T14:18:57,522 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/b70eaf8e714a4605b7b9d91707cac7eb, entries=150, sequenceid=458, filesize=12.0 K 2024-11-12T14:18:57,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/dbb40c0bdb1644148b606803bdef487c as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/dbb40c0bdb1644148b606803bdef487c 2024-11-12T14:18:57,526 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421197520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:57,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:57,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421197521, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:57,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:57,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:57,527 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:57,528 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/dbb40c0bdb1644148b606803bdef487c, entries=150, sequenceid=458, filesize=12.0 K 2024-11-12T14:18:57,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:57,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:57,529 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:57,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/b4de65f5894f4f52b03b03dcbb1c0557 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/b4de65f5894f4f52b03b03dcbb1c0557 2024-11-12T14:18:57,533 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/b4de65f5894f4f52b03b03dcbb1c0557, entries=150, sequenceid=458, filesize=12.0 K 2024-11-12T14:18:57,534 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 33074d5f6b8c90fb791f1838297603c6 in 185ms, sequenceid=458, compaction requested=false 2024-11-12T14:18:57,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:57,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:57,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-12T14:18:57,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-12T14:18:57,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-12T14:18:57,536 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 343 msec 2024-11-12T14:18:57,538 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T14:18:57,538 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 347 msec 2024-11-12T14:18:57,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:18:57,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:57,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:18:57,538 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:57,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:18:57,539 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:57,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:57,550 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/42f2d06f68484ccd9c5d20d14d02153b is 50, key is test_row_0/A:col10/1731421137499/Put/seqid=0 2024-11-12T14:18:57,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742344_1520 (size=14741) 2024-11-12T14:18:57,668 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:57,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421197659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:57,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:57,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:57,668 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:57,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:57,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421197668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:57,672 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:57,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:57,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421197668, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:57,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:57,672 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:57,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:57,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:57,672 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:57,786 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/d0183cc522c24542a2b1e40c98c0a28f as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/d0183cc522c24542a2b1e40c98c0a28f 2024-11-12T14:18:57,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:57,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421197779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:57,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:57,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:57,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421197779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:57,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:57,790 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:57,790 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:57,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421197779, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:57,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:57,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:57,790 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:57,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:57,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:57,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:57,792 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/B of 33074d5f6b8c90fb791f1838297603c6 into d0183cc522c24542a2b1e40c98c0a28f(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:57,792 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:57,792 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/B, priority=13, startTime=1731421137294; duration=0sec 2024-11-12T14:18:57,792 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:57,792 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:B 2024-11-12T14:18:57,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-12T14:18:57,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:57,819 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:18:57,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38668 deadline: 1731421197819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:57,820 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:18:57,820 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_2' in 33074d5f6b8c90fb791f1838297603c6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8276 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:18:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:57,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-12T14:18:57,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T14:18:57,821 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:18:57,822 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:18:57,822 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:18:57,836 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:57,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38622 deadline: 1731421197829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:57,836 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_2' in 33074d5f6b8c90fb791f1838297603c6 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8300 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:18:57,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:57,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:57,837 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:57,896 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/8b3310ae425a489383b6ea8b15b72b33 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/8b3310ae425a489383b6ea8b15b72b33 2024-11-12T14:18:57,901 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/C of 33074d5f6b8c90fb791f1838297603c6 into 8b3310ae425a489383b6ea8b15b72b33(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:57,901 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:57,901 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/C, priority=13, startTime=1731421137298; duration=0sec 2024-11-12T14:18:57,901 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:57,901 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:C 2024-11-12T14:18:57,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T14:18:57,962 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/42f2d06f68484ccd9c5d20d14d02153b 2024-11-12T14:18:57,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/417b1a273caa4ff59114b8ea33e439cc is 50, key is test_row_0/B:col10/1731421137499/Put/seqid=0 2024-11-12T14:18:57,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742345_1521 (size=12301) 2024-11-12T14:18:57,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-12T14:18:57,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:57,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:57,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:57,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:57,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:57,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:58,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:58,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421197999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:58,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:58,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421197999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:58,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:58,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:58,005 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:58,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:58,005 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:58,005 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:58,006 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:58,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421197999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:58,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:58,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:58,006 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:58,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-12T14:18:58,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:58,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:58,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:58,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:58,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:58,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:58,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T14:18:58,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-12T14:18:58,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:58,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:58,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:58,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:58,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:58,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:58,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:58,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421198309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:58,317 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:58,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421198309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:58,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:58,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:58,317 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:58,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:58,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:58,317 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:58,333 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:58,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421198320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:58,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:58,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:58,334 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:58,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/417b1a273caa4ff59114b8ea33e439cc 2024-11-12T14:18:58,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/943a01747a0d456fa1bf19f228df276a is 50, key is test_row_0/C:col10/1731421137499/Put/seqid=0 2024-11-12T14:18:58,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742346_1522 (size=12301) 2024-11-12T14:18:58,426 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=472 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/943a01747a0d456fa1bf19f228df276a 2024-11-12T14:18:58,434 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/42f2d06f68484ccd9c5d20d14d02153b as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/42f2d06f68484ccd9c5d20d14d02153b 2024-11-12T14:18:58,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-12T14:18:58,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:58,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:58,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:58,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:58,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:58,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:18:58,444 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/42f2d06f68484ccd9c5d20d14d02153b, entries=200, sequenceid=472, filesize=14.4 K 2024-11-12T14:18:58,446 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/417b1a273caa4ff59114b8ea33e439cc as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/417b1a273caa4ff59114b8ea33e439cc 2024-11-12T14:18:58,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T14:18:58,453 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/417b1a273caa4ff59114b8ea33e439cc, entries=150, sequenceid=472, filesize=12.0 K 2024-11-12T14:18:58,454 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/943a01747a0d456fa1bf19f228df276a as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/943a01747a0d456fa1bf19f228df276a 2024-11-12T14:18:58,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/943a01747a0d456fa1bf19f228df276a, entries=150, sequenceid=472, filesize=12.0 K 2024-11-12T14:18:58,464 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 33074d5f6b8c90fb791f1838297603c6 in 926ms, sequenceid=472, compaction requested=true 2024-11-12T14:18:58,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:58,464 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:58,465 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:58,465 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/A is initiating minor compaction (all files) 2024-11-12T14:18:58,465 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/A in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:58,465 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/7ad045ddd4df48929b3128b5697177c1, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/b70eaf8e714a4605b7b9d91707cac7eb, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/42f2d06f68484ccd9c5d20d14d02153b] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=39.4 K 2024-11-12T14:18:58,466 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7ad045ddd4df48929b3128b5697177c1, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731421135543 2024-11-12T14:18:58,466 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b70eaf8e714a4605b7b9d91707cac7eb, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1731421136820 2024-11-12T14:18:58,466 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 42f2d06f68484ccd9c5d20d14d02153b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1731421137499 2024-11-12T14:18:58,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:18:58,474 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:58,474 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:58,475 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:58,475 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/B is initiating minor compaction (all files) 2024-11-12T14:18:58,475 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/B in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:58,475 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/d0183cc522c24542a2b1e40c98c0a28f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/dbb40c0bdb1644148b606803bdef487c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/417b1a273caa4ff59114b8ea33e439cc] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=37.0 K 2024-11-12T14:18:58,475 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting d0183cc522c24542a2b1e40c98c0a28f, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731421135543 2024-11-12T14:18:58,475 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting dbb40c0bdb1644148b606803bdef487c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1731421136820 2024-11-12T14:18:58,476 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 417b1a273caa4ff59114b8ea33e439cc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1731421137499 2024-11-12T14:18:58,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:18:58,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:58,481 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:18:58,482 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:58,483 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#A#compaction#444 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:58,483 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/43c540bd57464352a274db31f7af91c9 is 50, key is test_row_0/A:col10/1731421137499/Put/seqid=0 2024-11-12T14:18:58,492 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#B#compaction#445 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:58,492 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/9722a64fbb224a16b9e5a6bc31ff4a8b is 50, key is test_row_0/B:col10/1731421137499/Put/seqid=0 2024-11-12T14:18:58,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742347_1523 (size=13391) 2024-11-12T14:18:58,539 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742348_1524 (size=13391) 2024-11-12T14:18:58,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-12T14:18:58,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:58,590 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T14:18:58,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:18:58,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:58,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:18:58,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:58,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:18:58,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:58,597 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/a3295c11aaf54c5398535e35bd042d09 is 50, key is test_row_0/A:col10/1731421137666/Put/seqid=0 2024-11-12T14:18:58,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742349_1525 (size=12301) 2024-11-12T14:18:58,610 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/a3295c11aaf54c5398535e35bd042d09 2024-11-12T14:18:58,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/bf1b955632a44c0db46e808a5868c0dd is 50, key is test_row_0/B:col10/1731421137666/Put/seqid=0 2024-11-12T14:18:58,653 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742350_1526 (size=12301) 2024-11-12T14:18:58,654 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/bf1b955632a44c0db46e808a5868c0dd 2024-11-12T14:18:58,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/e51ab5b981ac4b21887eea43cb30fa05 is 50, key is test_row_0/C:col10/1731421137666/Put/seqid=0 2024-11-12T14:18:58,679 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742351_1527 (size=12301) 2024-11-12T14:18:58,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:58,836 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:18:58,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:58,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421198859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:58,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:58,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:58,874 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:58,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:58,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421198866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:58,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:58,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:58,879 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:58,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:58,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421198869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:58,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:58,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:58,880 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:58,921 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T14:18:58,922 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/43c540bd57464352a274db31f7af91c9 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/43c540bd57464352a274db31f7af91c9 2024-11-12T14:18:58,925 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/A of 33074d5f6b8c90fb791f1838297603c6 into 43c540bd57464352a274db31f7af91c9(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:58,925 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:58,925 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/A, priority=13, startTime=1731421138464; duration=0sec 2024-11-12T14:18:58,925 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:58,925 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:A 2024-11-12T14:18:58,925 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:58,926 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:58,926 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/C is initiating minor compaction (all files) 2024-11-12T14:18:58,926 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/C in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:58,926 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/8b3310ae425a489383b6ea8b15b72b33, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/b4de65f5894f4f52b03b03dcbb1c0557, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/943a01747a0d456fa1bf19f228df276a] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=37.0 K 2024-11-12T14:18:58,926 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b3310ae425a489383b6ea8b15b72b33, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=438, earliestPutTs=1731421135543 2024-11-12T14:18:58,927 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4de65f5894f4f52b03b03dcbb1c0557, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=458, earliestPutTs=1731421136820 2024-11-12T14:18:58,927 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 943a01747a0d456fa1bf19f228df276a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1731421137499 2024-11-12T14:18:58,946 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#C#compaction#449 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:58,946 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/f900f9af876f4b048bac0a7d73f25306 is 50, key is test_row_0/C:col10/1731421137499/Put/seqid=0 2024-11-12T14:18:58,951 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/9722a64fbb224a16b9e5a6bc31ff4a8b as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/9722a64fbb224a16b9e5a6bc31ff4a8b 2024-11-12T14:18:58,956 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/B of 33074d5f6b8c90fb791f1838297603c6 into 9722a64fbb224a16b9e5a6bc31ff4a8b(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:58,956 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:58,956 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/B, priority=13, startTime=1731421138474; duration=0sec 2024-11-12T14:18:58,957 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:58,957 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:B 2024-11-12T14:18:58,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T14:18:58,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742352_1528 (size=13391) 2024-11-12T14:18:58,974 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/f900f9af876f4b048bac0a7d73f25306 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/f900f9af876f4b048bac0a7d73f25306 2024-11-12T14:18:58,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:58,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421198979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:58,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:58,982 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/C of 33074d5f6b8c90fb791f1838297603c6 into f900f9af876f4b048bac0a7d73f25306(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:58,982 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:58,982 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/C, priority=13, startTime=1731421138481; duration=0sec 2024-11-12T14:18:58,982 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:58,982 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:C 2024-11-12T14:18:58,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:58,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421198990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:58,994 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:58,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421198991, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:58,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:58,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:58,995 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:58,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:58,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:58,995 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:59,080 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=494 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/e51ab5b981ac4b21887eea43cb30fa05 2024-11-12T14:18:59,084 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/a3295c11aaf54c5398535e35bd042d09 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/a3295c11aaf54c5398535e35bd042d09 2024-11-12T14:18:59,089 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/a3295c11aaf54c5398535e35bd042d09, entries=150, sequenceid=494, filesize=12.0 K 2024-11-12T14:18:59,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/bf1b955632a44c0db46e808a5868c0dd as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/bf1b955632a44c0db46e808a5868c0dd 2024-11-12T14:18:59,093 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/bf1b955632a44c0db46e808a5868c0dd, entries=150, sequenceid=494, filesize=12.0 K 2024-11-12T14:18:59,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/e51ab5b981ac4b21887eea43cb30fa05 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/e51ab5b981ac4b21887eea43cb30fa05 2024-11-12T14:18:59,100 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/e51ab5b981ac4b21887eea43cb30fa05, entries=150, sequenceid=494, filesize=12.0 K 2024-11-12T14:18:59,101 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for 33074d5f6b8c90fb791f1838297603c6 in 511ms, sequenceid=494, compaction requested=false 2024-11-12T14:18:59,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:59,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:59,101 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-12T14:18:59,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-12T14:18:59,103 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-12T14:18:59,103 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2800 sec 2024-11-12T14:18:59,105 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.2840 sec 2024-11-12T14:18:59,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:59,191 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T14:18:59,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:18:59,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:59,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:18:59,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:59,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:18:59,191 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:18:59,195 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/7f933629ae8a46d0ab435bc3b914bd22 is 50, key is test_row_0/A:col10/1731421138835/Put/seqid=0 2024-11-12T14:18:59,202 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742353_1529 (size=14741) 2024-11-12T14:18:59,300 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:59,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421199283, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:59,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:59,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:59,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:59,316 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:59,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421199302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:59,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:59,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:59,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:59,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:59,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421199300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:59,319 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:59,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:59,320 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:59,417 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:59,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421199410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:59,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:59,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:59,418 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:59,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:59,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421199429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:59,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:59,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421199429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:59,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:59,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:59,435 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:59,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:59,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:59,435 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:59,607 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/7f933629ae8a46d0ab435bc3b914bd22 2024-11-12T14:18:59,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/da5adda77d2143b69c092febd09e0b76 is 50, key is test_row_0/B:col10/1731421138835/Put/seqid=0 2024-11-12T14:18:59,641 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:59,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421199638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:59,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:59,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421199639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:59,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:59,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:59,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 183 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421199639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:59,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:59,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:59,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:59,642 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:59,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:59,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:59,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:59,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:59,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742354_1530 (size=12301) 2024-11-12T14:18:59,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/da5adda77d2143b69c092febd09e0b76 2024-11-12T14:18:59,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/9e9b41c090a8413bab5975fdbb5bf377 is 50, key is test_row_0/C:col10/1731421138835/Put/seqid=0 2024-11-12T14:18:59,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742355_1531 (size=12301) 2024-11-12T14:18:59,678 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=510 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/9e9b41c090a8413bab5975fdbb5bf377 2024-11-12T14:18:59,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/7f933629ae8a46d0ab435bc3b914bd22 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/7f933629ae8a46d0ab435bc3b914bd22 2024-11-12T14:18:59,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/7f933629ae8a46d0ab435bc3b914bd22, entries=200, sequenceid=510, filesize=14.4 K 2024-11-12T14:18:59,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/da5adda77d2143b69c092febd09e0b76 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/da5adda77d2143b69c092febd09e0b76 2024-11-12T14:18:59,697 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/da5adda77d2143b69c092febd09e0b76, entries=150, sequenceid=510, filesize=12.0 K 2024-11-12T14:18:59,698 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/9e9b41c090a8413bab5975fdbb5bf377 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/9e9b41c090a8413bab5975fdbb5bf377 2024-11-12T14:18:59,701 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/9e9b41c090a8413bab5975fdbb5bf377, entries=150, sequenceid=510, filesize=12.0 K 2024-11-12T14:18:59,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=114.05 KB/116790 for 33074d5f6b8c90fb791f1838297603c6 in 511ms, sequenceid=510, compaction requested=true 2024-11-12T14:18:59,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:59,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:18:59,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:59,702 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:59,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:18:59,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:59,702 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:59,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:18:59,702 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:59,703 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:59,703 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/B is initiating minor compaction (all files) 2024-11-12T14:18:59,703 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/B in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:59,704 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/9722a64fbb224a16b9e5a6bc31ff4a8b, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/bf1b955632a44c0db46e808a5868c0dd, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/da5adda77d2143b69c092febd09e0b76] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=37.1 K 2024-11-12T14:18:59,704 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40433 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:59,704 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/A is initiating minor compaction (all files) 2024-11-12T14:18:59,704 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/A in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:59,704 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/43c540bd57464352a274db31f7af91c9, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/a3295c11aaf54c5398535e35bd042d09, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/7f933629ae8a46d0ab435bc3b914bd22] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=39.5 K 2024-11-12T14:18:59,704 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 9722a64fbb224a16b9e5a6bc31ff4a8b, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1731421137499 2024-11-12T14:18:59,704 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43c540bd57464352a274db31f7af91c9, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1731421137499 2024-11-12T14:18:59,704 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting bf1b955632a44c0db46e808a5868c0dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731421137654 2024-11-12T14:18:59,705 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3295c11aaf54c5398535e35bd042d09, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731421137654 2024-11-12T14:18:59,705 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f933629ae8a46d0ab435bc3b914bd22, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1731421138835 2024-11-12T14:18:59,706 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting da5adda77d2143b69c092febd09e0b76, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1731421138835 2024-11-12T14:18:59,739 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#A#compaction#453 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:59,739 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/e3c0a2c90f264a27a1e5ae9dc738f68d is 50, key is test_row_0/A:col10/1731421138835/Put/seqid=0 2024-11-12T14:18:59,748 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#B#compaction#454 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:59,748 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/68291904a3344c4abd8f1a40c3ff71c1 is 50, key is test_row_0/B:col10/1731421138835/Put/seqid=0 2024-11-12T14:18:59,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742357_1533 (size=13493) 2024-11-12T14:18:59,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742356_1532 (size=13493) 2024-11-12T14:18:59,777 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/68291904a3344c4abd8f1a40c3ff71c1 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/68291904a3344c4abd8f1a40c3ff71c1 2024-11-12T14:18:59,781 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/e3c0a2c90f264a27a1e5ae9dc738f68d as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/e3c0a2c90f264a27a1e5ae9dc738f68d 2024-11-12T14:18:59,788 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/A of 33074d5f6b8c90fb791f1838297603c6 into e3c0a2c90f264a27a1e5ae9dc738f68d(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:59,788 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/B of 33074d5f6b8c90fb791f1838297603c6 into 68291904a3344c4abd8f1a40c3ff71c1(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:59,788 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:59,788 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:59,788 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/B, priority=13, startTime=1731421139702; duration=0sec 2024-11-12T14:18:59,788 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/A, priority=13, startTime=1731421139702; duration=0sec 2024-11-12T14:18:59,789 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:59,789 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:B 2024-11-12T14:18:59,789 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:18:59,789 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:A 2024-11-12T14:18:59,789 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:18:59,789 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:18:59,790 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/C is initiating minor compaction (all files) 2024-11-12T14:18:59,790 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/C in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:18:59,790 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/f900f9af876f4b048bac0a7d73f25306, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/e51ab5b981ac4b21887eea43cb30fa05, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/9e9b41c090a8413bab5975fdbb5bf377] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=37.1 K 2024-11-12T14:18:59,790 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting f900f9af876f4b048bac0a7d73f25306, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=472, earliestPutTs=1731421137499 2024-11-12T14:18:59,790 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting e51ab5b981ac4b21887eea43cb30fa05, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=494, earliestPutTs=1731421137654 2024-11-12T14:18:59,790 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e9b41c090a8413bab5975fdbb5bf377, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1731421138835 2024-11-12T14:18:59,802 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#C#compaction#455 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:18:59,812 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/5c301c1d90204b8eaec82061d3c74272 is 50, key is test_row_0/C:col10/1731421138835/Put/seqid=0 2024-11-12T14:18:59,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742358_1534 (size=13493) 2024-11-12T14:18:59,852 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/5c301c1d90204b8eaec82061d3c74272 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/5c301c1d90204b8eaec82061d3c74272 2024-11-12T14:18:59,862 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/C of 33074d5f6b8c90fb791f1838297603c6 into 5c301c1d90204b8eaec82061d3c74272(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:18:59,862 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:18:59,862 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/C, priority=13, startTime=1731421139702; duration=0sec 2024-11-12T14:18:59,862 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:18:59,862 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:C 2024-11-12T14:18:59,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:18:59,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=120.76 KB heapSize=360.05 KB 2024-11-12T14:18:59,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:18:59,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:18:59,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:18:59,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:18:59,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:18:59,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:18:59,962 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/e3dda4ba62454665b892435d47b9e6b5 is 50, key is test_row_0/A:col10/1731421139299/Put/seqid=0 2024-11-12T14:18:59,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-12T14:18:59,970 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:18:59,971 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:18:59,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-12T14:18:59,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T14:18:59,973 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:18:59,974 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:18:59,974 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:18:59,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742359_1535 (size=14741) 2024-11-12T14:18:59,986 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/e3dda4ba62454665b892435d47b9e6b5 2024-11-12T14:18:59,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:18:59,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421199983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:18:59,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:59,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:18:59,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:18:59,995 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/eb12f2d528e240789f1ae95459d2af0d is 50, key is test_row_0/B:col10/1731421139299/Put/seqid=0 2024-11-12T14:19:00,003 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:00,003 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:00,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421199987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:00,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421199987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:00,003 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:19:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:00,004 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:19:00,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742360_1536 (size=12301) 2024-11-12T14:19:00,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/eb12f2d528e240789f1ae95459d2af0d 2024-11-12T14:19:00,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/8bad499e13d14ca1af87772b9fdfbffa is 50, key is test_row_0/C:col10/1731421139299/Put/seqid=0 2024-11-12T14:19:00,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742361_1537 (size=12301) 2024-11-12T14:19:00,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T14:19:00,107 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:00,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421200099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:00,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:00,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:00,108 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:19:00,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421200109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:00,121 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:00,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:00,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421200109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:00,121 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:00,122 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:19:00,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:00,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:00,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:19:00,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T14:19:00,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:00,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:19:00,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:00,128 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:00,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:00,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:00,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T14:19:00,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:00,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:19:00,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:00,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:00,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:00,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:00,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T14:19:00,323 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:00,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421200321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:00,324 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:19:00,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421200330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:00,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:00,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:00,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:00,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:19:00,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421200332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:00,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:00,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:00,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:19:00,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T14:19:00,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:00,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:19:00,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:00,436 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:00,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:00,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:00,469 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=534 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/8bad499e13d14ca1af87772b9fdfbffa 2024-11-12T14:19:00,472 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/e3dda4ba62454665b892435d47b9e6b5 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/e3dda4ba62454665b892435d47b9e6b5 2024-11-12T14:19:00,476 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/e3dda4ba62454665b892435d47b9e6b5, entries=200, sequenceid=534, filesize=14.4 K 2024-11-12T14:19:00,477 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/eb12f2d528e240789f1ae95459d2af0d as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/eb12f2d528e240789f1ae95459d2af0d 2024-11-12T14:19:00,480 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/eb12f2d528e240789f1ae95459d2af0d, entries=150, sequenceid=534, filesize=12.0 K 2024-11-12T14:19:00,481 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/8bad499e13d14ca1af87772b9fdfbffa as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/8bad499e13d14ca1af87772b9fdfbffa 2024-11-12T14:19:00,485 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/8bad499e13d14ca1af87772b9fdfbffa, entries=150, sequenceid=534, filesize=12.0 K 2024-11-12T14:19:00,486 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for 33074d5f6b8c90fb791f1838297603c6 in 530ms, sequenceid=534, compaction requested=false 2024-11-12T14:19:00,486 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:19:00,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T14:19:00,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-12T14:19:00,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:00,604 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=53.67 KB heapSize=160.13 KB 2024-11-12T14:19:00,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:19:00,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:00,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:19:00,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:00,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:19:00,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:00,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/71ca1929e60b43efaf01676c0feab515 is 50, key is test_row_0/A:col10/1731421139981/Put/seqid=0 2024-11-12T14:19:00,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:19:00,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:19:00,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742362_1538 (size=12301) 2024-11-12T14:19:00,668 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=545 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/71ca1929e60b43efaf01676c0feab515 2024-11-12T14:19:00,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/7c88baffa9e34044a81a254c18170cb2 is 50, key is test_row_0/B:col10/1731421139981/Put/seqid=0 2024-11-12T14:19:00,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731421140694, max timestamp=1731421140726Num uniques 859; 2024-11-12T14:19:00,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731421140694, max timestamp=1731421140726Num uniques 859; 2024-11-12T14:19:00,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-4 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T14:19:00,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-0 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T14:19:00,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-9 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=209, cellSize=9565, totalHeapSize=21976, min timestamp=1731421140694, max timestamp=1731421140726Num uniques 859; 2024-11-12T14:19:00,729 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-9 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=9565, suffix heap size=117064,new segment heap size=21976 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=209 2024-11-12T14:19:00,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742363_1539 (size=12301) 2024-11-12T14:19:00,740 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=545 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/7c88baffa9e34044a81a254c18170cb2 2024-11-12T14:19:00,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/cb09328e88fd437c8f1df7e19ff5f020 is 50, key is test_row_0/C:col10/1731421139981/Put/seqid=0 2024-11-12T14:19:00,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742364_1540 (size=12301) 2024-11-12T14:19:00,779 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=545 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/cb09328e88fd437c8f1df7e19ff5f020 2024-11-12T14:19:00,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/71ca1929e60b43efaf01676c0feab515 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/71ca1929e60b43efaf01676c0feab515 2024-11-12T14:19:00,793 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/71ca1929e60b43efaf01676c0feab515, entries=150, sequenceid=545, filesize=12.0 K 2024-11-12T14:19:00,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/7c88baffa9e34044a81a254c18170cb2 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/7c88baffa9e34044a81a254c18170cb2 2024-11-12T14:19:00,799 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/7c88baffa9e34044a81a254c18170cb2, entries=150, sequenceid=545, filesize=12.0 K 2024-11-12T14:19:00,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/cb09328e88fd437c8f1df7e19ff5f020 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/cb09328e88fd437c8f1df7e19ff5f020 2024-11-12T14:19:00,808 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/cb09328e88fd437c8f1df7e19ff5f020, entries=150, sequenceid=545, filesize=12.0 K 2024-11-12T14:19:00,813 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~160.08 KB/163920, currentSize=107.34 KB/109920 for 33074d5f6b8c90fb791f1838297603c6 in 208ms, sequenceid=545, compaction requested=true 2024-11-12T14:19:00,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:19:00,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:19:00,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:00,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-12T14:19:00,813 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=114.05 KB heapSize=320.53 KB 2024-11-12T14:19:00,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:19:00,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:19:00,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:19:00,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:19:00,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:19:00,813 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:19:00,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-12T14:19:00,816 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-12T14:19:00,816 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 840 msec 2024-11-12T14:19:00,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/534f617b25494271a2dcc606f95ad23a is 50, key is test_row_0/A:col10/1731421140805/Put/seqid=0 2024-11-12T14:19:00,818 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 846 msec 2024-11-12T14:19:00,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742365_1541 (size=14741) 2024-11-12T14:19:00,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=578 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/534f617b25494271a2dcc606f95ad23a 2024-11-12T14:19:00,857 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/71d397e64154407c8074c500f5335f8b is 50, key is test_row_0/B:col10/1731421140805/Put/seqid=0 2024-11-12T14:19:00,900 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:00,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421200882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:00,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:00,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:00,901 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:19:00,902 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742366_1542 (size=12301) 2024-11-12T14:19:00,902 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=578 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/71d397e64154407c8074c500f5335f8b 2024-11-12T14:19:00,910 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:00,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 203 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421200897, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:00,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:00,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:00,911 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:19:00,918 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:00,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 180 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421200901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:00,918 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:00,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:00,919 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:19:00,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/5c168544d760467a879b6a8155479bed is 50, key is test_row_0/C:col10/1731421140805/Put/seqid=0 2024-11-12T14:19:00,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742367_1543 (size=12301) 2024-11-12T14:19:01,024 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:01,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421201012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:01,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:01,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 204 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421201019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:01,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:01,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:01,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:19:01,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:01,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:01,027 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:19:01,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:01,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 181 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421201029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:01,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:01,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:01,045 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:19:01,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-12T14:19:01,109 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:19:01,110 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:19:01,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-12T14:19:01,112 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:19:01,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T14:19:01,114 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:19:01,114 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:19:01,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T14:19:01,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421201230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:01,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:01,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:01,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:01,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 205 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421201230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:01,237 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:19:01,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:01,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:01,238 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:19:01,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 182 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421201251, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:01,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:01,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:01,261 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:19:01,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T14:19:01,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:01,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:19:01,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:01,270 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:01,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:01,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:01,359 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=578 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/5c168544d760467a879b6a8155479bed 2024-11-12T14:19:01,370 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/534f617b25494271a2dcc606f95ad23a as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/534f617b25494271a2dcc606f95ad23a 2024-11-12T14:19:01,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/534f617b25494271a2dcc606f95ad23a, entries=200, sequenceid=578, filesize=14.4 K 2024-11-12T14:19:01,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/71d397e64154407c8074c500f5335f8b as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/71d397e64154407c8074c500f5335f8b 2024-11-12T14:19:01,380 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/71d397e64154407c8074c500f5335f8b, entries=150, sequenceid=578, filesize=12.0 K 2024-11-12T14:19:01,381 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/5c168544d760467a879b6a8155479bed as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/5c168544d760467a879b6a8155479bed 2024-11-12T14:19:01,390 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/5c168544d760467a879b6a8155479bed, entries=150, sequenceid=578, filesize=12.0 K 2024-11-12T14:19:01,408 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~320.48 KB/328176, currentSize=67.09 KB/68700 for 33074d5f6b8c90fb791f1838297603c6 in 595ms, sequenceid=578, compaction requested=true 2024-11-12T14:19:01,409 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:19:01,409 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:19:01,413 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55276 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:19:01,413 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/A is initiating minor compaction (all files) 2024-11-12T14:19:01,413 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/A in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:01,413 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/e3c0a2c90f264a27a1e5ae9dc738f68d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/e3dda4ba62454665b892435d47b9e6b5, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/71ca1929e60b43efaf01676c0feab515, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/534f617b25494271a2dcc606f95ad23a] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=54.0 K 2024-11-12T14:19:01,414 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3c0a2c90f264a27a1e5ae9dc738f68d, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1731421138835 2024-11-12T14:19:01,415 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting e3dda4ba62454665b892435d47b9e6b5, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731421139281 2024-11-12T14:19:01,415 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71ca1929e60b43efaf01676c0feab515, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=545, earliestPutTs=1731421139981 2024-11-12T14:19:01,415 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 534f617b25494271a2dcc606f95ad23a, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=578, earliestPutTs=1731421140771 2024-11-12T14:19:01,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:19:01,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:01,423 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:19:01,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:19:01,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:01,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:19:01,424 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:19:01,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-12T14:19:01,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:01,425 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T14:19:01,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:19:01,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:01,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:19:01,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:01,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:19:01,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:01,428 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#A#compaction#465 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:01,429 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/e8e401a94ca041818293e2153448d48f is 50, key is test_row_0/A:col10/1731421140805/Put/seqid=0 2024-11-12T14:19:01,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T14:19:01,430 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50396 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:19:01,430 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/B is initiating minor compaction (all files) 2024-11-12T14:19:01,430 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/B in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:01,430 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/68291904a3344c4abd8f1a40c3ff71c1, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/eb12f2d528e240789f1ae95459d2af0d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/7c88baffa9e34044a81a254c18170cb2, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/71d397e64154407c8074c500f5335f8b] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=49.2 K 2024-11-12T14:19:01,431 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 68291904a3344c4abd8f1a40c3ff71c1, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1731421138835 2024-11-12T14:19:01,431 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting eb12f2d528e240789f1ae95459d2af0d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731421139293 2024-11-12T14:19:01,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/16440ae38e4e4d16b46856080226380a is 50, key is test_row_0/A:col10/1731421140819/Put/seqid=0 2024-11-12T14:19:01,435 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c88baffa9e34044a81a254c18170cb2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=545, earliestPutTs=1731421139981 2024-11-12T14:19:01,436 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 71d397e64154407c8074c500f5335f8b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=578, earliestPutTs=1731421140771 2024-11-12T14:19:01,459 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#B#compaction#467 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:01,460 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/6a7fe7702c3e439fa5e85ed0c810dcac is 50, key is test_row_0/B:col10/1731421140805/Put/seqid=0 2024-11-12T14:19:01,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742368_1544 (size=13629) 2024-11-12T14:19:01,478 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/e8e401a94ca041818293e2153448d48f as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/e8e401a94ca041818293e2153448d48f 2024-11-12T14:19:01,486 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742369_1545 (size=12301) 2024-11-12T14:19:01,489 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/16440ae38e4e4d16b46856080226380a 2024-11-12T14:19:01,494 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/A of 33074d5f6b8c90fb791f1838297603c6 into e8e401a94ca041818293e2153448d48f(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:19:01,494 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:19:01,494 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/A, priority=12, startTime=1731421141409; duration=0sec 2024-11-12T14:19:01,494 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:19:01,494 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:A 2024-11-12T14:19:01,496 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:19:01,498 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50396 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:19:01,498 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/C is initiating minor compaction (all files) 2024-11-12T14:19:01,498 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/C in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:01,498 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/5c301c1d90204b8eaec82061d3c74272, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/8bad499e13d14ca1af87772b9fdfbffa, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/cb09328e88fd437c8f1df7e19ff5f020, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/5c168544d760467a879b6a8155479bed] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=49.2 K 2024-11-12T14:19:01,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/9a9f72a62a7e45efb74e202c335f31dd is 50, key is test_row_0/B:col10/1731421140819/Put/seqid=0 2024-11-12T14:19:01,520 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c301c1d90204b8eaec82061d3c74272, keycount=150, bloomtype=ROW, size=13.2 K, encoding=NONE, compression=NONE, seqNum=510, earliestPutTs=1731421138835 2024-11-12T14:19:01,521 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bad499e13d14ca1af87772b9fdfbffa, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=534, earliestPutTs=1731421139293 2024-11-12T14:19:01,521 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb09328e88fd437c8f1df7e19ff5f020, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=545, earliestPutTs=1731421139981 2024-11-12T14:19:01,521 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c168544d760467a879b6a8155479bed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=578, earliestPutTs=1731421140771 2024-11-12T14:19:01,537 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742370_1546 (size=13629) 2024-11-12T14:19:01,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742371_1547 (size=12301) 2024-11-12T14:19:01,542 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/9a9f72a62a7e45efb74e202c335f31dd 2024-11-12T14:19:01,547 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#C#compaction#469 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:01,548 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/d952ca34c76941768609215a4f5fddbb is 50, key is test_row_0/C:col10/1731421140805/Put/seqid=0 2024-11-12T14:19:01,550 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/6a7fe7702c3e439fa5e85ed0c810dcac as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/6a7fe7702c3e439fa5e85ed0c810dcac 2024-11-12T14:19:01,556 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/B of 33074d5f6b8c90fb791f1838297603c6 into 6a7fe7702c3e439fa5e85ed0c810dcac(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:19:01,556 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:19:01,556 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/B, priority=12, startTime=1731421141422; duration=0sec 2024-11-12T14:19:01,556 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:01,556 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:B 2024-11-12T14:19:01,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:19:01,558 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:19:01,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/b62d296d42c744708babcba0012d23d2 is 50, key is test_row_0/C:col10/1731421140819/Put/seqid=0 2024-11-12T14:19:01,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742372_1548 (size=13629) 2024-11-12T14:19:01,579 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/d952ca34c76941768609215a4f5fddbb as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/d952ca34c76941768609215a4f5fddbb 2024-11-12T14:19:01,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742373_1549 (size=12301) 2024-11-12T14:19:01,587 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/C of 33074d5f6b8c90fb791f1838297603c6 into d952ca34c76941768609215a4f5fddbb(size=13.3 K), total size for store is 13.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:19:01,587 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:19:01,587 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/C, priority=12, startTime=1731421141424; duration=0sec 2024-11-12T14:19:01,587 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:01,587 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:C 2024-11-12T14:19:01,588 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=591 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/b62d296d42c744708babcba0012d23d2 2024-11-12T14:19:01,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/16440ae38e4e4d16b46856080226380a as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/16440ae38e4e4d16b46856080226380a 2024-11-12T14:19:01,609 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/16440ae38e4e4d16b46856080226380a, entries=150, sequenceid=591, filesize=12.0 K 2024-11-12T14:19:01,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/9a9f72a62a7e45efb74e202c335f31dd as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/9a9f72a62a7e45efb74e202c335f31dd 2024-11-12T14:19:01,613 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/9a9f72a62a7e45efb74e202c335f31dd, entries=150, sequenceid=591, filesize=12.0 K 2024-11-12T14:19:01,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/b62d296d42c744708babcba0012d23d2 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/b62d296d42c744708babcba0012d23d2 2024-11-12T14:19:01,620 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/b62d296d42c744708babcba0012d23d2, entries=150, sequenceid=591, filesize=12.0 K 2024-11-12T14:19:01,621 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=87.22 KB/89310 for 33074d5f6b8c90fb791f1838297603c6 in 196ms, sequenceid=591, compaction requested=false 2024-11-12T14:19:01,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:19:01,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:01,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-12T14:19:01,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-12T14:19:01,624 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-12T14:19:01,624 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 509 msec 2024-11-12T14:19:01,626 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 515 msec 2024-11-12T14:19:01,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T14:19:01,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:19:01,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:01,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:19:01,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:01,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:19:01,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:01,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:19:01,633 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/a3cab3ef57204f30a2f021d2502604dd is 50, key is test_row_0/A:col10/1731421141626/Put/seqid=0 2024-11-12T14:19:01,659 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742374_1550 (size=12301) 2024-11-12T14:19:01,669 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=612 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/a3cab3ef57204f30a2f021d2502604dd 2024-11-12T14:19:01,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:01,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 190 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421201667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:01,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/74573e18dc6c4c0c965bb5dbd2c8883e is 50, key is test_row_0/B:col10/1731421141626/Put/seqid=0 2024-11-12T14:19:01,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:01,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:01,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:19:01,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:01,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421201669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:01,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:01,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:01,683 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:19:01,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:01,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 216 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421201671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:01,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:01,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:01,690 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:19:01,694 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742375_1551 (size=12301) 2024-11-12T14:19:01,695 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=612 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/74573e18dc6c4c0c965bb5dbd2c8883e 2024-11-12T14:19:01,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-12T14:19:01,739 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:19:01,740 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:19:01,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-12T14:19:01,741 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/0ad488e0c76f42a89a090d331acbdcd2 is 50, key is test_row_0/C:col10/1731421141626/Put/seqid=0 2024-11-12T14:19:01,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T14:19:01,744 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:19:01,748 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:19:01,749 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:19:01,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:01,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 191 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421201784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:01,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:01,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:01,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:19:01,802 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:01,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421201791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:01,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:01,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:01,802 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:19:01,805 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742376_1552 (size=12301) 2024-11-12T14:19:01,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:01,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 217 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421201800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:01,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:01,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:01,807 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:19:01,808 INFO [Thread-1945 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1945. 2024-11-12T14:19:01,808 DEBUG [Thread-1945 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:19:01,808 DEBUG [Thread-1945 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:01,808 DEBUG [Thread-1945 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:01,808 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:19:01,810 INFO [Thread-1937 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1937. 2024-11-12T14:19:01,810 DEBUG [Thread-1937 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:19:01,810 DEBUG [Thread-1937 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:01,811 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:19:01,811 DEBUG [Thread-1937 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:01,821 INFO [Thread-1943 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1943. 2024-11-12T14:19:01,821 DEBUG [Thread-1943 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:19:01,821 DEBUG [Thread-1943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:01,821 DEBUG [Thread-1943 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:01,821 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:19:01,823 INFO [Thread-1941 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1941. 2024-11-12T14:19:01,823 DEBUG [Thread-1941 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:19:01,823 DEBUG [Thread-1941 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:01,823 DEBUG [Thread-1941 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:01,823 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:19:01,826 INFO [Thread-1939 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1939. 2024-11-12T14:19:01,826 DEBUG [Thread-1939 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:19:01,826 DEBUG [Thread-1939 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:01,826 DEBUG [Thread-1939 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:01,826 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:19:01,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T14:19:01,907 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T14:19:01,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:01,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:19:01,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:01,908 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:01,908 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:01,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:01,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:02,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 192 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38640 deadline: 1731421201999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:02,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:19:02,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:02,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38658 deadline: 1731421202009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:02,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:02,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:02,010 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:19:02,019 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:02,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 218 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38654 deadline: 1731421202019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:02,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 , the old value is region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:02,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:02,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., hostname=2b6d221c5cde,42925,1731420990822, seqNum=2 because the exception is null or not the one we care about 2024-11-12T14:19:02,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T14:19:02,060 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T14:19:02,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:02,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:19:02,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:02,060 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:02,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:02,206 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=612 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/0ad488e0c76f42a89a090d331acbdcd2 2024-11-12T14:19:02,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/a3cab3ef57204f30a2f021d2502604dd as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/a3cab3ef57204f30a2f021d2502604dd 2024-11-12T14:19:02,212 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/a3cab3ef57204f30a2f021d2502604dd, entries=150, sequenceid=612, filesize=12.0 K 2024-11-12T14:19:02,212 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T14:19:02,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:02,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:19:02,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:02,213 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:02,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:02,213 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/74573e18dc6c4c0c965bb5dbd2c8883e as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/74573e18dc6c4c0c965bb5dbd2c8883e 2024-11-12T14:19:02,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:02,216 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/74573e18dc6c4c0c965bb5dbd2c8883e, entries=150, sequenceid=612, filesize=12.0 K 2024-11-12T14:19:02,217 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/0ad488e0c76f42a89a090d331acbdcd2 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/0ad488e0c76f42a89a090d331acbdcd2 2024-11-12T14:19:02,220 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/0ad488e0c76f42a89a090d331acbdcd2, entries=150, sequenceid=612, filesize=12.0 K 2024-11-12T14:19:02,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for 33074d5f6b8c90fb791f1838297603c6 in 593ms, sequenceid=612, compaction requested=true 2024-11-12T14:19:02,221 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:19:02,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:19:02,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:02,221 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:19:02,221 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:19:02,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:19:02,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:02,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 33074d5f6b8c90fb791f1838297603c6:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:19:02,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:19:02,224 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:19:02,224 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:19:02,224 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/B is initiating minor compaction (all files) 2024-11-12T14:19:02,224 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/A is initiating minor compaction (all files) 2024-11-12T14:19:02,224 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/B in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:02,224 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/A in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:02,224 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/6a7fe7702c3e439fa5e85ed0c810dcac, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/9a9f72a62a7e45efb74e202c335f31dd, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/74573e18dc6c4c0c965bb5dbd2c8883e] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=37.3 K 2024-11-12T14:19:02,225 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/e8e401a94ca041818293e2153448d48f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/16440ae38e4e4d16b46856080226380a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/a3cab3ef57204f30a2f021d2502604dd] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=37.3 K 2024-11-12T14:19:02,225 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting e8e401a94ca041818293e2153448d48f, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=578, earliestPutTs=1731421140771 2024-11-12T14:19:02,225 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 6a7fe7702c3e439fa5e85ed0c810dcac, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=578, earliestPutTs=1731421140771 2024-11-12T14:19:02,225 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16440ae38e4e4d16b46856080226380a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1731421140819 2024-11-12T14:19:02,225 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a9f72a62a7e45efb74e202c335f31dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1731421140819 2024-11-12T14:19:02,226 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3cab3ef57204f30a2f021d2502604dd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=612, earliestPutTs=1731421141606 2024-11-12T14:19:02,226 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 74573e18dc6c4c0c965bb5dbd2c8883e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=612, earliestPutTs=1731421141606 2024-11-12T14:19:02,236 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#A#compaction#474 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:02,236 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/2315c529f2004de98c23255b2ef4811b is 50, key is test_row_0/A:col10/1731421141626/Put/seqid=0 2024-11-12T14:19:02,236 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#B#compaction#475 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:02,237 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/78208bb9e6314482b67310b1f4955be6 is 50, key is test_row_0/B:col10/1731421141626/Put/seqid=0 2024-11-12T14:19:02,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742377_1553 (size=13731) 2024-11-12T14:19:02,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742378_1554 (size=13731) 2024-11-12T14:19:02,244 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/2315c529f2004de98c23255b2ef4811b as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/2315c529f2004de98c23255b2ef4811b 2024-11-12T14:19:02,244 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/78208bb9e6314482b67310b1f4955be6 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/78208bb9e6314482b67310b1f4955be6 2024-11-12T14:19:02,249 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/B of 33074d5f6b8c90fb791f1838297603c6 into 78208bb9e6314482b67310b1f4955be6(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:19:02,249 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/A of 33074d5f6b8c90fb791f1838297603c6 into 2315c529f2004de98c23255b2ef4811b(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:19:02,249 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:19:02,249 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:19:02,249 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/B, priority=13, startTime=1731421142221; duration=0sec 2024-11-12T14:19:02,249 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/A, priority=13, startTime=1731421142221; duration=0sec 2024-11-12T14:19:02,249 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:19:02,249 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:A 2024-11-12T14:19:02,249 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:19:02,249 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:B 2024-11-12T14:19:02,249 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:19:02,251 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38231 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:19:02,251 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): 33074d5f6b8c90fb791f1838297603c6/C is initiating minor compaction (all files) 2024-11-12T14:19:02,251 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 33074d5f6b8c90fb791f1838297603c6/C in TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:02,251 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/d952ca34c76941768609215a4f5fddbb, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/b62d296d42c744708babcba0012d23d2, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/0ad488e0c76f42a89a090d331acbdcd2] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp, totalSize=37.3 K 2024-11-12T14:19:02,252 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting d952ca34c76941768609215a4f5fddbb, keycount=150, bloomtype=ROW, size=13.3 K, encoding=NONE, compression=NONE, seqNum=578, earliestPutTs=1731421140771 2024-11-12T14:19:02,252 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b62d296d42c744708babcba0012d23d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=591, earliestPutTs=1731421140819 2024-11-12T14:19:02,252 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ad488e0c76f42a89a090d331acbdcd2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=612, earliestPutTs=1731421141606 2024-11-12T14:19:02,260 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 33074d5f6b8c90fb791f1838297603c6#C#compaction#476 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:02,261 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/0f0aaa3bf5284e9c81f1f0962db3efa7 is 50, key is test_row_0/C:col10/1731421141626/Put/seqid=0 2024-11-12T14:19:02,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742379_1555 (size=13731) 2024-11-12T14:19:02,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:19:02,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T14:19:02,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:19:02,311 INFO [Thread-1932 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1932. 2024-11-12T14:19:02,311 DEBUG [Thread-1932 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:19:02,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:02,311 DEBUG [Thread-1932 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:02,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:19:02,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:02,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:19:02,311 DEBUG [Thread-1932 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:02,311 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:02,311 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:19:02,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/945ca0582e6244e9b739b52459f08789 is 50, key is test_row_0/A:col10/1731421142310/Put/seqid=0 2024-11-12T14:19:02,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742380_1556 (size=12301) 2024-11-12T14:19:02,320 INFO [Thread-1930 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1930. 2024-11-12T14:19:02,320 DEBUG [Thread-1930 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:19:02,320 DEBUG [Thread-1930 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:02,320 DEBUG [Thread-1930 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:02,320 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:19:02,330 INFO [Thread-1934 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1934. 2024-11-12T14:19:02,330 DEBUG [Thread-1934 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:19:02,330 DEBUG [Thread-1934 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:02,330 DEBUG [Thread-1934 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:02,330 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:19:02,365 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T14:19:02,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:02,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:19:02,365 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:02,365 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:02,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:02,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:02,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T14:19:02,518 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T14:19:02,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:02,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:19:02,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:02,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:02,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:02,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:02,668 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/0f0aaa3bf5284e9c81f1f0962db3efa7 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/0f0aaa3bf5284e9c81f1f0962db3efa7 2024-11-12T14:19:02,670 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T14:19:02,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:02,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:19:02,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:02,671 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:02,671 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:02,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:02,671 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 33074d5f6b8c90fb791f1838297603c6/C of 33074d5f6b8c90fb791f1838297603c6 into 0f0aaa3bf5284e9c81f1f0962db3efa7(size=13.4 K), total size for store is 13.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:19:02,671 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:19:02,672 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., storeName=33074d5f6b8c90fb791f1838297603c6/C, priority=13, startTime=1731421142221; duration=0sec 2024-11-12T14:19:02,672 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:02,672 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 33074d5f6b8c90fb791f1838297603c6:C 2024-11-12T14:19:02,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=629 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/945ca0582e6244e9b739b52459f08789 2024-11-12T14:19:02,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/288fd8511f724a4fa2a4f537b9a3433f is 50, key is test_row_0/B:col10/1731421142310/Put/seqid=0 2024-11-12T14:19:02,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742381_1557 (size=12301) 2024-11-12T14:19:02,823 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T14:19:02,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:02,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:19:02,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:02,823 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:02,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:02,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:02,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T14:19:02,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T14:19:02,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:02,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:19:02,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:02,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:02,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:02,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:03,128 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=629 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/288fd8511f724a4fa2a4f537b9a3433f 2024-11-12T14:19:03,128 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T14:19:03,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:03,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:19:03,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:03,129 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:03,129 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:03,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:03,133 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/e967dec29b304bba813596f1f01f0425 is 50, key is test_row_0/C:col10/1731421142310/Put/seqid=0 2024-11-12T14:19:03,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742382_1558 (size=12301) 2024-11-12T14:19:03,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T14:19:03,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:03,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:19:03,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:03,282 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:03,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:03,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:03,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T14:19:03,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:03,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. as already flushing 2024-11-12T14:19:03,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:03,435 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:03,435 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:03,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:03,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=629 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/e967dec29b304bba813596f1f01f0425 2024-11-12T14:19:03,541 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/945ca0582e6244e9b739b52459f08789 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/945ca0582e6244e9b739b52459f08789 2024-11-12T14:19:03,544 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/945ca0582e6244e9b739b52459f08789, entries=150, sequenceid=629, filesize=12.0 K 2024-11-12T14:19:03,545 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/288fd8511f724a4fa2a4f537b9a3433f as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/288fd8511f724a4fa2a4f537b9a3433f 2024-11-12T14:19:03,547 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/288fd8511f724a4fa2a4f537b9a3433f, entries=150, sequenceid=629, filesize=12.0 K 2024-11-12T14:19:03,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/e967dec29b304bba813596f1f01f0425 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/e967dec29b304bba813596f1f01f0425 2024-11-12T14:19:03,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/e967dec29b304bba813596f1f01f0425, entries=150, sequenceid=629, filesize=12.0 K 2024-11-12T14:19:03,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=13.42 KB/13740 for 33074d5f6b8c90fb791f1838297603c6 in 1240ms, sequenceid=629, compaction requested=false 2024-11-12T14:19:03,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:19:03,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-12T14:19:03,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:03,587 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-12T14:19:03,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:19:03,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:03,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:19:03,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:03,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:19:03,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:03,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/4d06985952804ddb921d64dde07cf44f is 50, key is test_row_1/A:col10/1731421142319/Put/seqid=0 2024-11-12T14:19:03,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742383_1559 (size=9857) 2024-11-12T14:19:03,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T14:19:03,994 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/4d06985952804ddb921d64dde07cf44f 2024-11-12T14:19:04,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/6f7cc3132bac471fac44172acc1e6389 is 50, key is test_row_1/B:col10/1731421142319/Put/seqid=0 2024-11-12T14:19:04,007 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742384_1560 (size=9857) 2024-11-12T14:19:04,008 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/6f7cc3132bac471fac44172acc1e6389 2024-11-12T14:19:04,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/9b069898847148748c31d753ea47b041 is 50, key is test_row_1/C:col10/1731421142319/Put/seqid=0 2024-11-12T14:19:04,016 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742385_1561 (size=9857) 2024-11-12T14:19:04,017 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=635 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/9b069898847148748c31d753ea47b041 2024-11-12T14:19:04,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/4d06985952804ddb921d64dde07cf44f as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/4d06985952804ddb921d64dde07cf44f 2024-11-12T14:19:04,024 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/4d06985952804ddb921d64dde07cf44f, entries=100, sequenceid=635, filesize=9.6 K 2024-11-12T14:19:04,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/6f7cc3132bac471fac44172acc1e6389 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/6f7cc3132bac471fac44172acc1e6389 2024-11-12T14:19:04,029 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/6f7cc3132bac471fac44172acc1e6389, entries=100, sequenceid=635, filesize=9.6 K 2024-11-12T14:19:04,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/9b069898847148748c31d753ea47b041 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/9b069898847148748c31d753ea47b041 2024-11-12T14:19:04,034 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/9b069898847148748c31d753ea47b041, entries=100, sequenceid=635, filesize=9.6 K 2024-11-12T14:19:04,035 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 33074d5f6b8c90fb791f1838297603c6 in 448ms, sequenceid=635, compaction requested=true 2024-11-12T14:19:04,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 33074d5f6b8c90fb791f1838297603c6: 2024-11-12T14:19:04,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:04,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-12T14:19:04,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-12T14:19:04,038 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-12T14:19:04,038 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2870 sec 2024-11-12T14:19:04,039 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 2.2980 sec 2024-11-12T14:19:05,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-12T14:19:05,910 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:19:07,880 INFO [Thread-1928 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1928. 2024-11-12T14:19:07,880 INFO [Thread-1926 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1926. 2024-11-12T14:19:07,880 DEBUG [Thread-1926 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:19:07,880 DEBUG [Thread-1928 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:19:07,880 DEBUG [Thread-1926 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:07,880 DEBUG [Thread-1928 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:07,880 DEBUG [Thread-1926 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:07,880 DEBUG [Thread-1928 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:07,880 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:19:07,880 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:19:07,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T14:19:07,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-12T14:19:07,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 49 2024-11-12T14:19:07,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 124 2024-11-12T14:19:07,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 131 2024-11-12T14:19:07,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 152 2024-11-12T14:19:07,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T14:19:07,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T14:19:07,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 916 2024-11-12T14:19:07,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2747 rows 2024-11-12T14:19:07,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 913 2024-11-12T14:19:07,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2739 rows 2024-11-12T14:19:07,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 914 2024-11-12T14:19:07,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2735 rows 2024-11-12T14:19:07,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 916 2024-11-12T14:19:07,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2747 rows 2024-11-12T14:19:07,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 913 2024-11-12T14:19:07,881 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2739 rows 2024-11-12T14:19:07,881 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T14:19:07,881 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T14:19:07,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:07,881 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:07,881 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:19:07,882 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-12T14:19:07,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T14:19:07,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T14:19:07,884 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731421147884"}]},"ts":"1731421147884"} 2024-11-12T14:19:07,886 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T14:19:07,886 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T14:19:07,886 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T14:19:07,887 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=33074d5f6b8c90fb791f1838297603c6, UNASSIGN}] 2024-11-12T14:19:07,889 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=33074d5f6b8c90fb791f1838297603c6, UNASSIGN 2024-11-12T14:19:07,890 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=33074d5f6b8c90fb791f1838297603c6, regionState=CLOSING, regionLocation=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:07,892 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=133, ppid=132, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=33074d5f6b8c90fb791f1838297603c6, UNASSIGN because future has completed 2024-11-12T14:19:07,892 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T14:19:07,892 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure 33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822}] 2024-11-12T14:19:07,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T14:19:08,045 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(122): Close 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:19:08,045 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T14:19:08,045 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1722): Closing 33074d5f6b8c90fb791f1838297603c6, disabling compactions & flushes 2024-11-12T14:19:08,045 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:08,045 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:08,045 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. after waiting 0 ms 2024-11-12T14:19:08,045 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:08,045 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(2902): Flushing 33074d5f6b8c90fb791f1838297603c6 3/3 column families, dataSize=13.42 KB heapSize=40.59 KB 2024-11-12T14:19:08,046 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=A 2024-11-12T14:19:08,046 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:08,046 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=B 2024-11-12T14:19:08,046 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:08,046 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactingMemStore(205): FLUSHING TO DISK 33074d5f6b8c90fb791f1838297603c6, store=C 2024-11-12T14:19:08,046 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:08,050 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/a4786cb7472640f5a553a1e23f9f594f is 50, key is test_row_2/A:col10/1731421147879/Put/seqid=0 2024-11-12T14:19:08,063 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742386_1562 (size=7415) 2024-11-12T14:19:08,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T14:19:08,463 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=640 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/a4786cb7472640f5a553a1e23f9f594f 2024-11-12T14:19:08,482 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/1622c12926a94531a52babc073ed2a38 is 50, key is test_row_2/B:col10/1731421147879/Put/seqid=0 2024-11-12T14:19:08,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742387_1563 (size=7415) 2024-11-12T14:19:08,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T14:19:08,904 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=640 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/1622c12926a94531a52babc073ed2a38 2024-11-12T14:19:08,911 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/72c5030ce9a34f178e5b1e0dc59cb1df is 50, key is test_row_2/C:col10/1731421147879/Put/seqid=0 2024-11-12T14:19:08,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742388_1564 (size=7415) 2024-11-12T14:19:09,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T14:19:09,318 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=640 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/72c5030ce9a34f178e5b1e0dc59cb1df 2024-11-12T14:19:09,322 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/A/a4786cb7472640f5a553a1e23f9f594f as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/a4786cb7472640f5a553a1e23f9f594f 2024-11-12T14:19:09,324 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/a4786cb7472640f5a553a1e23f9f594f, entries=50, sequenceid=640, filesize=7.2 K 2024-11-12T14:19:09,325 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/B/1622c12926a94531a52babc073ed2a38 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/1622c12926a94531a52babc073ed2a38 2024-11-12T14:19:09,327 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/1622c12926a94531a52babc073ed2a38, entries=50, sequenceid=640, filesize=7.2 K 2024-11-12T14:19:09,328 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/.tmp/C/72c5030ce9a34f178e5b1e0dc59cb1df as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/72c5030ce9a34f178e5b1e0dc59cb1df 2024-11-12T14:19:09,330 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/72c5030ce9a34f178e5b1e0dc59cb1df, entries=50, sequenceid=640, filesize=7.2 K 2024-11-12T14:19:09,331 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 33074d5f6b8c90fb791f1838297603c6 in 1286ms, sequenceid=640, compaction requested=true 2024-11-12T14:19:09,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/3c021a86f0fa48c7aead303b5250e346, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/289bb14d29db4984a404908620d4f3b2, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/95b1d08c39a54e90982b67926820508b, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/959b6757d6154f3d981d16987004e83e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/378cd1d63cef4db184dc522c17debeee, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/07f0f4e2a7ff44e9a8717cd94a2624e4, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/063bc55d18c14851b945c25fe1867c31, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/7c653b9d40b946ec82cfc45b65832caa, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/5175e3a00a33414b99a15cc5fdf2f2d2, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/17f07ef41935475fbee71e8cf8bf6990, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/ed4ef2d5d0da43229322bd8d80d6f850, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/c401cb2cbd974ae0b01845d6fd2670b1, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/dae343df2400410d9f133575924dcdcd, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/a7b31d0bdde94f908877cb0e876270b6, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/1105bee6c0fd40aca38f76a91d225adc, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/d70a7cb9d7e043ef9d49c27a6abac988, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/96b7b8eae907406c9e117f29a8d82dea, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/5ef3858cda4c4a9891362161cc9146f2, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/71c81f2c17c14d7885c26906cc7628c0, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/26481f14c7b046a1b6f6ac9a68c4ae14, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/40542fd13bb040938caee14c2bb5be69, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/82affaad25c04e39846fa8cb02bfe232, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/4f91e0838e5246fd81631879b126bf99, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/62a3011546514d0ba3762e89b0811c09, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/32a110ccba734597ad2622128f59d786, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/3123d4c7670d495dab3c8f7314afbe46, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/63270c58329e4ad092e1a1c65eb022cd, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/3de296d5ebb34c499dee98daea87faa3, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/87006a1eed204fa9b9e91eab7fb90046, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/7ad045ddd4df48929b3128b5697177c1, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/b70eaf8e714a4605b7b9d91707cac7eb, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/42f2d06f68484ccd9c5d20d14d02153b, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/43c540bd57464352a274db31f7af91c9, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/a3295c11aaf54c5398535e35bd042d09, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/7f933629ae8a46d0ab435bc3b914bd22, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/e3c0a2c90f264a27a1e5ae9dc738f68d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/e3dda4ba62454665b892435d47b9e6b5, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/71ca1929e60b43efaf01676c0feab515, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/534f617b25494271a2dcc606f95ad23a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/e8e401a94ca041818293e2153448d48f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/16440ae38e4e4d16b46856080226380a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/a3cab3ef57204f30a2f021d2502604dd] to archive 2024-11-12T14:19:09,332 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T14:19:09,334 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/3c021a86f0fa48c7aead303b5250e346 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/3c021a86f0fa48c7aead303b5250e346 2024-11-12T14:19:09,335 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/289bb14d29db4984a404908620d4f3b2 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/289bb14d29db4984a404908620d4f3b2 2024-11-12T14:19:09,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/95b1d08c39a54e90982b67926820508b to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/95b1d08c39a54e90982b67926820508b 2024-11-12T14:19:09,336 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/959b6757d6154f3d981d16987004e83e to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/959b6757d6154f3d981d16987004e83e 2024-11-12T14:19:09,337 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/378cd1d63cef4db184dc522c17debeee to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/378cd1d63cef4db184dc522c17debeee 2024-11-12T14:19:09,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/07f0f4e2a7ff44e9a8717cd94a2624e4 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/07f0f4e2a7ff44e9a8717cd94a2624e4 2024-11-12T14:19:09,338 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/063bc55d18c14851b945c25fe1867c31 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/063bc55d18c14851b945c25fe1867c31 2024-11-12T14:19:09,339 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/7c653b9d40b946ec82cfc45b65832caa to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/7c653b9d40b946ec82cfc45b65832caa 2024-11-12T14:19:09,340 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/5175e3a00a33414b99a15cc5fdf2f2d2 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/5175e3a00a33414b99a15cc5fdf2f2d2 2024-11-12T14:19:09,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/17f07ef41935475fbee71e8cf8bf6990 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/17f07ef41935475fbee71e8cf8bf6990 2024-11-12T14:19:09,341 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/ed4ef2d5d0da43229322bd8d80d6f850 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/ed4ef2d5d0da43229322bd8d80d6f850 2024-11-12T14:19:09,342 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/c401cb2cbd974ae0b01845d6fd2670b1 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/c401cb2cbd974ae0b01845d6fd2670b1 2024-11-12T14:19:09,343 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/dae343df2400410d9f133575924dcdcd to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/dae343df2400410d9f133575924dcdcd 2024-11-12T14:19:09,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/a7b31d0bdde94f908877cb0e876270b6 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/a7b31d0bdde94f908877cb0e876270b6 2024-11-12T14:19:09,344 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/1105bee6c0fd40aca38f76a91d225adc to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/1105bee6c0fd40aca38f76a91d225adc 2024-11-12T14:19:09,345 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/d70a7cb9d7e043ef9d49c27a6abac988 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/d70a7cb9d7e043ef9d49c27a6abac988 2024-11-12T14:19:09,346 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/96b7b8eae907406c9e117f29a8d82dea to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/96b7b8eae907406c9e117f29a8d82dea 2024-11-12T14:19:09,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/5ef3858cda4c4a9891362161cc9146f2 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/5ef3858cda4c4a9891362161cc9146f2 2024-11-12T14:19:09,347 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/71c81f2c17c14d7885c26906cc7628c0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/71c81f2c17c14d7885c26906cc7628c0 2024-11-12T14:19:09,348 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/26481f14c7b046a1b6f6ac9a68c4ae14 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/26481f14c7b046a1b6f6ac9a68c4ae14 2024-11-12T14:19:09,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/40542fd13bb040938caee14c2bb5be69 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/40542fd13bb040938caee14c2bb5be69 2024-11-12T14:19:09,349 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/82affaad25c04e39846fa8cb02bfe232 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/82affaad25c04e39846fa8cb02bfe232 2024-11-12T14:19:09,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/4f91e0838e5246fd81631879b126bf99 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/4f91e0838e5246fd81631879b126bf99 2024-11-12T14:19:09,350 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/62a3011546514d0ba3762e89b0811c09 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/62a3011546514d0ba3762e89b0811c09 2024-11-12T14:19:09,351 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/32a110ccba734597ad2622128f59d786 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/32a110ccba734597ad2622128f59d786 2024-11-12T14:19:09,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/3123d4c7670d495dab3c8f7314afbe46 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/3123d4c7670d495dab3c8f7314afbe46 2024-11-12T14:19:09,352 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/63270c58329e4ad092e1a1c65eb022cd to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/63270c58329e4ad092e1a1c65eb022cd 2024-11-12T14:19:09,353 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/3de296d5ebb34c499dee98daea87faa3 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/3de296d5ebb34c499dee98daea87faa3 2024-11-12T14:19:09,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/87006a1eed204fa9b9e91eab7fb90046 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/87006a1eed204fa9b9e91eab7fb90046 2024-11-12T14:19:09,354 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/7ad045ddd4df48929b3128b5697177c1 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/7ad045ddd4df48929b3128b5697177c1 2024-11-12T14:19:09,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/b70eaf8e714a4605b7b9d91707cac7eb to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/b70eaf8e714a4605b7b9d91707cac7eb 2024-11-12T14:19:09,355 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/42f2d06f68484ccd9c5d20d14d02153b to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/42f2d06f68484ccd9c5d20d14d02153b 2024-11-12T14:19:09,356 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/43c540bd57464352a274db31f7af91c9 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/43c540bd57464352a274db31f7af91c9 2024-11-12T14:19:09,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/a3295c11aaf54c5398535e35bd042d09 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/a3295c11aaf54c5398535e35bd042d09 2024-11-12T14:19:09,357 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/7f933629ae8a46d0ab435bc3b914bd22 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/7f933629ae8a46d0ab435bc3b914bd22 2024-11-12T14:19:09,358 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/e3c0a2c90f264a27a1e5ae9dc738f68d to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/e3c0a2c90f264a27a1e5ae9dc738f68d 2024-11-12T14:19:09,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/e3dda4ba62454665b892435d47b9e6b5 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/e3dda4ba62454665b892435d47b9e6b5 2024-11-12T14:19:09,359 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/71ca1929e60b43efaf01676c0feab515 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/71ca1929e60b43efaf01676c0feab515 2024-11-12T14:19:09,360 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/534f617b25494271a2dcc606f95ad23a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/534f617b25494271a2dcc606f95ad23a 2024-11-12T14:19:09,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/e8e401a94ca041818293e2153448d48f to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/e8e401a94ca041818293e2153448d48f 2024-11-12T14:19:09,361 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/16440ae38e4e4d16b46856080226380a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/16440ae38e4e4d16b46856080226380a 2024-11-12T14:19:09,362 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/a3cab3ef57204f30a2f021d2502604dd to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/a3cab3ef57204f30a2f021d2502604dd 2024-11-12T14:19:09,363 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/0b2a1dd58bdb41b6a5df074e53996cea, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/c7f6d777c75f4205812f0ab40bf47312, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/8406dac22c064be6a25f8fba349372c9, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/23d7b9616a5f42a499bc88cb269f4149, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/041574a1d65242b894af49159dfa98b8, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/8a40607ae8084ff1a7b2c999ebcac0a4, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/faf6f92af730404580a80027a7735e7d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/042d5d5aa0cc4a3ca15777a540dadf3d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/97765a8484db45ea9b0481be18ed9295, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/a7e2196866bb478eae731f18dcc6cf55, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/6ca8c2e9ffee43a2bc752db1b7c25483, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/c9ba1bb4261349b2bb1b203a30ae6e7f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/f8d75caced5e4b85bcd63bfba79a00ef, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/a773a506c46e4fb9839ef2551fd20583, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/0e52e25c0d4e4104964c7b48db5865dc, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/2a4958c66c1f42a4a0c84381b9ab9af9, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/6d20ecb93ed1404bb9a6844e01db91f6, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/cd6d782800df458ca7c6ad5f86fbf4d8, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/984a8404d36b47b69bae5ba17f92cde6, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/b7f4aaa3ed4043a8b0d87326126a10be, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/fedf0ceb3f4547a28a415db80ff389fc, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/8cf6f8e15c82429daeeb603dfa64a055, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/593ab6e5137f4ec7adaf9f254baa99bf, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/0c3e6de1bffe41908197a5d598d86646, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/7396fcffc53046a4bd4bb72e097f0ad5, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/87de762ab69d42b29e647eb544c53ce2, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/7a2bf67f29304ab69c5519bd104cc9d1, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/986b81e336514f3da51f19a76e08749f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/d0183cc522c24542a2b1e40c98c0a28f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/528b03a311ee4cf5b19807bfe669b928, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/dbb40c0bdb1644148b606803bdef487c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/9722a64fbb224a16b9e5a6bc31ff4a8b, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/417b1a273caa4ff59114b8ea33e439cc, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/bf1b955632a44c0db46e808a5868c0dd, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/68291904a3344c4abd8f1a40c3ff71c1, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/da5adda77d2143b69c092febd09e0b76, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/eb12f2d528e240789f1ae95459d2af0d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/7c88baffa9e34044a81a254c18170cb2, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/6a7fe7702c3e439fa5e85ed0c810dcac, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/71d397e64154407c8074c500f5335f8b, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/9a9f72a62a7e45efb74e202c335f31dd, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/74573e18dc6c4c0c965bb5dbd2c8883e] to archive 2024-11-12T14:19:09,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T14:19:09,364 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/0b2a1dd58bdb41b6a5df074e53996cea to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/0b2a1dd58bdb41b6a5df074e53996cea 2024-11-12T14:19:09,365 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/c7f6d777c75f4205812f0ab40bf47312 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/c7f6d777c75f4205812f0ab40bf47312 2024-11-12T14:19:09,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/8406dac22c064be6a25f8fba349372c9 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/8406dac22c064be6a25f8fba349372c9 2024-11-12T14:19:09,366 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/23d7b9616a5f42a499bc88cb269f4149 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/23d7b9616a5f42a499bc88cb269f4149 2024-11-12T14:19:09,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/041574a1d65242b894af49159dfa98b8 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/041574a1d65242b894af49159dfa98b8 2024-11-12T14:19:09,367 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/8a40607ae8084ff1a7b2c999ebcac0a4 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/8a40607ae8084ff1a7b2c999ebcac0a4 2024-11-12T14:19:09,368 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/faf6f92af730404580a80027a7735e7d to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/faf6f92af730404580a80027a7735e7d 2024-11-12T14:19:09,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/042d5d5aa0cc4a3ca15777a540dadf3d to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/042d5d5aa0cc4a3ca15777a540dadf3d 2024-11-12T14:19:09,369 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/97765a8484db45ea9b0481be18ed9295 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/97765a8484db45ea9b0481be18ed9295 2024-11-12T14:19:09,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/a7e2196866bb478eae731f18dcc6cf55 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/a7e2196866bb478eae731f18dcc6cf55 2024-11-12T14:19:09,370 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/6ca8c2e9ffee43a2bc752db1b7c25483 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/6ca8c2e9ffee43a2bc752db1b7c25483 2024-11-12T14:19:09,371 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/c9ba1bb4261349b2bb1b203a30ae6e7f to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/c9ba1bb4261349b2bb1b203a30ae6e7f 2024-11-12T14:19:09,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/f8d75caced5e4b85bcd63bfba79a00ef to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/f8d75caced5e4b85bcd63bfba79a00ef 2024-11-12T14:19:09,372 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/a773a506c46e4fb9839ef2551fd20583 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/a773a506c46e4fb9839ef2551fd20583 2024-11-12T14:19:09,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/0e52e25c0d4e4104964c7b48db5865dc to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/0e52e25c0d4e4104964c7b48db5865dc 2024-11-12T14:19:09,373 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/2a4958c66c1f42a4a0c84381b9ab9af9 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/2a4958c66c1f42a4a0c84381b9ab9af9 2024-11-12T14:19:09,374 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/6d20ecb93ed1404bb9a6844e01db91f6 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/6d20ecb93ed1404bb9a6844e01db91f6 2024-11-12T14:19:09,375 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/cd6d782800df458ca7c6ad5f86fbf4d8 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/cd6d782800df458ca7c6ad5f86fbf4d8 2024-11-12T14:19:09,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/984a8404d36b47b69bae5ba17f92cde6 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/984a8404d36b47b69bae5ba17f92cde6 2024-11-12T14:19:09,376 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/b7f4aaa3ed4043a8b0d87326126a10be to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/b7f4aaa3ed4043a8b0d87326126a10be 2024-11-12T14:19:09,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/fedf0ceb3f4547a28a415db80ff389fc to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/fedf0ceb3f4547a28a415db80ff389fc 2024-11-12T14:19:09,377 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/8cf6f8e15c82429daeeb603dfa64a055 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/8cf6f8e15c82429daeeb603dfa64a055 2024-11-12T14:19:09,378 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/593ab6e5137f4ec7adaf9f254baa99bf to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/593ab6e5137f4ec7adaf9f254baa99bf 2024-11-12T14:19:09,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/0c3e6de1bffe41908197a5d598d86646 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/0c3e6de1bffe41908197a5d598d86646 2024-11-12T14:19:09,379 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/7396fcffc53046a4bd4bb72e097f0ad5 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/7396fcffc53046a4bd4bb72e097f0ad5 2024-11-12T14:19:09,380 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/87de762ab69d42b29e647eb544c53ce2 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/87de762ab69d42b29e647eb544c53ce2 2024-11-12T14:19:09,381 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/7a2bf67f29304ab69c5519bd104cc9d1 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/7a2bf67f29304ab69c5519bd104cc9d1 2024-11-12T14:19:09,382 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/986b81e336514f3da51f19a76e08749f to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/986b81e336514f3da51f19a76e08749f 2024-11-12T14:19:09,383 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/d0183cc522c24542a2b1e40c98c0a28f to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/d0183cc522c24542a2b1e40c98c0a28f 2024-11-12T14:19:09,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/528b03a311ee4cf5b19807bfe669b928 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/528b03a311ee4cf5b19807bfe669b928 2024-11-12T14:19:09,384 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/dbb40c0bdb1644148b606803bdef487c to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/dbb40c0bdb1644148b606803bdef487c 2024-11-12T14:19:09,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/9722a64fbb224a16b9e5a6bc31ff4a8b to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/9722a64fbb224a16b9e5a6bc31ff4a8b 2024-11-12T14:19:09,385 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/417b1a273caa4ff59114b8ea33e439cc to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/417b1a273caa4ff59114b8ea33e439cc 2024-11-12T14:19:09,386 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/bf1b955632a44c0db46e808a5868c0dd to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/bf1b955632a44c0db46e808a5868c0dd 2024-11-12T14:19:09,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/68291904a3344c4abd8f1a40c3ff71c1 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/68291904a3344c4abd8f1a40c3ff71c1 2024-11-12T14:19:09,387 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/da5adda77d2143b69c092febd09e0b76 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/da5adda77d2143b69c092febd09e0b76 2024-11-12T14:19:09,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/eb12f2d528e240789f1ae95459d2af0d to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/eb12f2d528e240789f1ae95459d2af0d 2024-11-12T14:19:09,388 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/7c88baffa9e34044a81a254c18170cb2 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/7c88baffa9e34044a81a254c18170cb2 2024-11-12T14:19:09,389 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/6a7fe7702c3e439fa5e85ed0c810dcac to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/6a7fe7702c3e439fa5e85ed0c810dcac 2024-11-12T14:19:09,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/71d397e64154407c8074c500f5335f8b to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/71d397e64154407c8074c500f5335f8b 2024-11-12T14:19:09,390 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/9a9f72a62a7e45efb74e202c335f31dd to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/9a9f72a62a7e45efb74e202c335f31dd 2024-11-12T14:19:09,391 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/74573e18dc6c4c0c965bb5dbd2c8883e to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/74573e18dc6c4c0c965bb5dbd2c8883e 2024-11-12T14:19:09,392 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/f72f91d0aba34d66bef60673686bb892, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/2683857647134d86b00263ec9c80dc44, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/e3099985d2ae40b59b6a1890af491060, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/933ac913575946e39abefce6750352a9, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/5ebb079e8d3e461ca3c37736eab690e2, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/d1d95649162241b8a025e1404d971fd7, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/e0a17bdbaf594b1ea86283684879f2db, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/47848300f17540c28f0c0ad05370deed, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/d49edaafe55f46b3a550e5b18f61a5c0, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/825d0278cec34f4eae3f38d48c7880c4, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/d4716fd012c046f2a953a87c8ff0fe33, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/b710ef911d71482ca4f8ea5a406440f2, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/9e5c8472b4174a8680b1b1e5f035f1ec, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/cdca18a5fc16438592e1bbe1c675d62d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/40004a1ec9ee43e78911a97ce7e24293, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/42f383bccb12469ea531d35295dd3adf, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/3ecbc83fffcb48eabcceceef22317535, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/52b64e4c174a45e38dc3b6a9f77ffb1e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/f9aafe01f08e41acac9ac8d114becf2a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/b55b0d27f1bf4c6eac5110d7e20290b8, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/268e2f5a17954459a5325ebc83fc1c8f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/441aac4aed7a4f2b973e1f8ce1734fb4, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/bbf492a2b2454b93bd5149cc3ae55523, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/f8b345bc14494f27930872337f7c801b, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/7c18747ed60f40f188b8f753c4eef919, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/254fa06fb4294e2a940e2382acbc7ee8, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/36eb1aa6901f49e6b60863f5bb17deef, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/c991fe6ce335435ca9a6419d1b8d5d35, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/8b3310ae425a489383b6ea8b15b72b33, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/7033efc722ce43b2aa60c9316ea82c83, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/b4de65f5894f4f52b03b03dcbb1c0557, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/f900f9af876f4b048bac0a7d73f25306, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/943a01747a0d456fa1bf19f228df276a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/e51ab5b981ac4b21887eea43cb30fa05, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/5c301c1d90204b8eaec82061d3c74272, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/9e9b41c090a8413bab5975fdbb5bf377, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/8bad499e13d14ca1af87772b9fdfbffa, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/cb09328e88fd437c8f1df7e19ff5f020, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/d952ca34c76941768609215a4f5fddbb, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/5c168544d760467a879b6a8155479bed, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/b62d296d42c744708babcba0012d23d2, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/0ad488e0c76f42a89a090d331acbdcd2] to archive 2024-11-12T14:19:09,393 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T14:19:09,394 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/f72f91d0aba34d66bef60673686bb892 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/f72f91d0aba34d66bef60673686bb892 2024-11-12T14:19:09,395 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/2683857647134d86b00263ec9c80dc44 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/2683857647134d86b00263ec9c80dc44 2024-11-12T14:19:09,396 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/e3099985d2ae40b59b6a1890af491060 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/e3099985d2ae40b59b6a1890af491060 2024-11-12T14:19:09,397 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/933ac913575946e39abefce6750352a9 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/933ac913575946e39abefce6750352a9 2024-11-12T14:19:09,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/5ebb079e8d3e461ca3c37736eab690e2 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/5ebb079e8d3e461ca3c37736eab690e2 2024-11-12T14:19:09,398 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/d1d95649162241b8a025e1404d971fd7 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/d1d95649162241b8a025e1404d971fd7 2024-11-12T14:19:09,399 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/e0a17bdbaf594b1ea86283684879f2db to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/e0a17bdbaf594b1ea86283684879f2db 2024-11-12T14:19:09,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/47848300f17540c28f0c0ad05370deed to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/47848300f17540c28f0c0ad05370deed 2024-11-12T14:19:09,400 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/d49edaafe55f46b3a550e5b18f61a5c0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/d49edaafe55f46b3a550e5b18f61a5c0 2024-11-12T14:19:09,401 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/825d0278cec34f4eae3f38d48c7880c4 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/825d0278cec34f4eae3f38d48c7880c4 2024-11-12T14:19:09,402 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/d4716fd012c046f2a953a87c8ff0fe33 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/d4716fd012c046f2a953a87c8ff0fe33 2024-11-12T14:19:09,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/b710ef911d71482ca4f8ea5a406440f2 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/b710ef911d71482ca4f8ea5a406440f2 2024-11-12T14:19:09,403 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/9e5c8472b4174a8680b1b1e5f035f1ec to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/9e5c8472b4174a8680b1b1e5f035f1ec 2024-11-12T14:19:09,404 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/cdca18a5fc16438592e1bbe1c675d62d to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/cdca18a5fc16438592e1bbe1c675d62d 2024-11-12T14:19:09,405 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/40004a1ec9ee43e78911a97ce7e24293 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/40004a1ec9ee43e78911a97ce7e24293 2024-11-12T14:19:09,406 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/42f383bccb12469ea531d35295dd3adf to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/42f383bccb12469ea531d35295dd3adf 2024-11-12T14:19:09,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/3ecbc83fffcb48eabcceceef22317535 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/3ecbc83fffcb48eabcceceef22317535 2024-11-12T14:19:09,407 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/52b64e4c174a45e38dc3b6a9f77ffb1e to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/52b64e4c174a45e38dc3b6a9f77ffb1e 2024-11-12T14:19:09,408 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/f9aafe01f08e41acac9ac8d114becf2a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/f9aafe01f08e41acac9ac8d114becf2a 2024-11-12T14:19:09,409 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/b55b0d27f1bf4c6eac5110d7e20290b8 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/b55b0d27f1bf4c6eac5110d7e20290b8 2024-11-12T14:19:09,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/268e2f5a17954459a5325ebc83fc1c8f to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/268e2f5a17954459a5325ebc83fc1c8f 2024-11-12T14:19:09,410 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/441aac4aed7a4f2b973e1f8ce1734fb4 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/441aac4aed7a4f2b973e1f8ce1734fb4 2024-11-12T14:19:09,411 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/bbf492a2b2454b93bd5149cc3ae55523 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/bbf492a2b2454b93bd5149cc3ae55523 2024-11-12T14:19:09,412 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/f8b345bc14494f27930872337f7c801b to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/f8b345bc14494f27930872337f7c801b 2024-11-12T14:19:09,413 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/7c18747ed60f40f188b8f753c4eef919 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/7c18747ed60f40f188b8f753c4eef919 2024-11-12T14:19:09,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/254fa06fb4294e2a940e2382acbc7ee8 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/254fa06fb4294e2a940e2382acbc7ee8 2024-11-12T14:19:09,414 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/36eb1aa6901f49e6b60863f5bb17deef to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/36eb1aa6901f49e6b60863f5bb17deef 2024-11-12T14:19:09,415 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/c991fe6ce335435ca9a6419d1b8d5d35 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/c991fe6ce335435ca9a6419d1b8d5d35 2024-11-12T14:19:09,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/8b3310ae425a489383b6ea8b15b72b33 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/8b3310ae425a489383b6ea8b15b72b33 2024-11-12T14:19:09,416 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/7033efc722ce43b2aa60c9316ea82c83 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/7033efc722ce43b2aa60c9316ea82c83 2024-11-12T14:19:09,417 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/b4de65f5894f4f52b03b03dcbb1c0557 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/b4de65f5894f4f52b03b03dcbb1c0557 2024-11-12T14:19:09,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/f900f9af876f4b048bac0a7d73f25306 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/f900f9af876f4b048bac0a7d73f25306 2024-11-12T14:19:09,418 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/943a01747a0d456fa1bf19f228df276a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/943a01747a0d456fa1bf19f228df276a 2024-11-12T14:19:09,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/e51ab5b981ac4b21887eea43cb30fa05 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/e51ab5b981ac4b21887eea43cb30fa05 2024-11-12T14:19:09,419 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/5c301c1d90204b8eaec82061d3c74272 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/5c301c1d90204b8eaec82061d3c74272 2024-11-12T14:19:09,420 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/9e9b41c090a8413bab5975fdbb5bf377 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/9e9b41c090a8413bab5975fdbb5bf377 2024-11-12T14:19:09,421 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/8bad499e13d14ca1af87772b9fdfbffa to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/8bad499e13d14ca1af87772b9fdfbffa 2024-11-12T14:19:09,422 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/cb09328e88fd437c8f1df7e19ff5f020 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/cb09328e88fd437c8f1df7e19ff5f020 2024-11-12T14:19:09,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/d952ca34c76941768609215a4f5fddbb to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/d952ca34c76941768609215a4f5fddbb 2024-11-12T14:19:09,423 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/5c168544d760467a879b6a8155479bed to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/5c168544d760467a879b6a8155479bed 2024-11-12T14:19:09,424 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/b62d296d42c744708babcba0012d23d2 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/b62d296d42c744708babcba0012d23d2 2024-11-12T14:19:09,425 DEBUG [StoreCloser-TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/0ad488e0c76f42a89a090d331acbdcd2 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/0ad488e0c76f42a89a090d331acbdcd2 2024-11-12T14:19:09,428 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/recovered.edits/643.seqid, newMaxSeqId=643, maxSeqId=1 2024-11-12T14:19:09,429 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. 2024-11-12T14:19:09,429 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] regionserver.HRegion(1676): Region close journal for 33074d5f6b8c90fb791f1838297603c6: Waiting for close lock at 1731421148045Running coprocessor pre-close hooks at 1731421148045Disabling compacts and flushes for region at 1731421148045Disabling writes for close at 1731421148045Obtaining lock to block concurrent updates at 1731421148045Preparing flush snapshotting stores in 33074d5f6b8c90fb791f1838297603c6 at 1731421148045Finished memstore snapshotting TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=41520, getOffHeapSize=0, getCellsCount=300 at 1731421148046 (+1 ms)Flushing stores of TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6. at 1731421148046Flushing 33074d5f6b8c90fb791f1838297603c6/A: creating writer at 1731421148046Flushing 33074d5f6b8c90fb791f1838297603c6/A: appending metadata at 1731421148049 (+3 ms)Flushing 33074d5f6b8c90fb791f1838297603c6/A: closing flushed file at 1731421148049Flushing 33074d5f6b8c90fb791f1838297603c6/B: creating writer at 1731421148475 (+426 ms)Flushing 33074d5f6b8c90fb791f1838297603c6/B: appending metadata at 1731421148481 (+6 ms)Flushing 33074d5f6b8c90fb791f1838297603c6/B: closing flushed file at 1731421148481Flushing 33074d5f6b8c90fb791f1838297603c6/C: creating writer at 1731421148908 (+427 ms)Flushing 33074d5f6b8c90fb791f1838297603c6/C: appending metadata at 1731421148910 (+2 ms)Flushing 33074d5f6b8c90fb791f1838297603c6/C: closing flushed file at 1731421148910Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@568ee07c: reopening flushed file at 1731421149321 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@a5a1ddc: reopening flushed file at 1731421149324 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@132a359b: reopening flushed file at 1731421149327 (+3 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~40.55 KB/41520, currentSize=0 B/0 for 33074d5f6b8c90fb791f1838297603c6 in 1286ms, sequenceid=640, compaction requested=true at 1731421149331 (+4 ms)Writing region close event to WAL at 1731421149426 (+95 ms)Running coprocessor post-close hooks at 1731421149429 (+3 ms)Closed at 1731421149429 2024-11-12T14:19:09,430 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=134}] handler.UnassignRegionHandler(157): Closed 33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:19:09,431 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=133 updating hbase:meta row=33074d5f6b8c90fb791f1838297603c6, regionState=CLOSED 2024-11-12T14:19:09,432 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=134, ppid=133, state=RUNNABLE, hasLock=false; CloseRegionProcedure 33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 because future has completed 2024-11-12T14:19:09,433 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-12T14:19:09,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; CloseRegionProcedure 33074d5f6b8c90fb791f1838297603c6, server=2b6d221c5cde,42925,1731420990822 in 1.5400 sec 2024-11-12T14:19:09,435 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=133, resume processing ppid=132 2024-11-12T14:19:09,435 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, ppid=132, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=33074d5f6b8c90fb791f1838297603c6, UNASSIGN in 1.5470 sec 2024-11-12T14:19:09,436 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-12T14:19:09,436 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5490 sec 2024-11-12T14:19:09,437 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731421149437"}]},"ts":"1731421149437"} 2024-11-12T14:19:09,438 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T14:19:09,438 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T14:19:09,440 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5570 sec 2024-11-12T14:19:10,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-12T14:19:10,030 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:19:10,031 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-12T14:19:10,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:19:10,038 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=135, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:19:10,039 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=135, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:19:10,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-12T14:19:10,048 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:19:10,051 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A, FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B, FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C, FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/recovered.edits] 2024-11-12T14:19:10,057 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/2315c529f2004de98c23255b2ef4811b to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/2315c529f2004de98c23255b2ef4811b 2024-11-12T14:19:10,059 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/4d06985952804ddb921d64dde07cf44f to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/4d06985952804ddb921d64dde07cf44f 2024-11-12T14:19:10,066 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/945ca0582e6244e9b739b52459f08789 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/945ca0582e6244e9b739b52459f08789 2024-11-12T14:19:10,070 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/a4786cb7472640f5a553a1e23f9f594f to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/A/a4786cb7472640f5a553a1e23f9f594f 2024-11-12T14:19:10,090 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/1622c12926a94531a52babc073ed2a38 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/1622c12926a94531a52babc073ed2a38 2024-11-12T14:19:10,097 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/288fd8511f724a4fa2a4f537b9a3433f to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/288fd8511f724a4fa2a4f537b9a3433f 2024-11-12T14:19:10,098 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/6f7cc3132bac471fac44172acc1e6389 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/6f7cc3132bac471fac44172acc1e6389 2024-11-12T14:19:10,100 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/78208bb9e6314482b67310b1f4955be6 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/B/78208bb9e6314482b67310b1f4955be6 2024-11-12T14:19:10,102 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/0f0aaa3bf5284e9c81f1f0962db3efa7 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/0f0aaa3bf5284e9c81f1f0962db3efa7 2024-11-12T14:19:10,103 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/72c5030ce9a34f178e5b1e0dc59cb1df to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/72c5030ce9a34f178e5b1e0dc59cb1df 2024-11-12T14:19:10,104 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/9b069898847148748c31d753ea47b041 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/9b069898847148748c31d753ea47b041 2024-11-12T14:19:10,105 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/e967dec29b304bba813596f1f01f0425 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/C/e967dec29b304bba813596f1f01f0425 2024-11-12T14:19:10,107 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/recovered.edits/643.seqid to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6/recovered.edits/643.seqid 2024-11-12T14:19:10,107 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/33074d5f6b8c90fb791f1838297603c6 2024-11-12T14:19:10,108 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T14:19:10,117 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=135, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:19:10,120 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T14:19:10,126 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T14:19:10,129 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=135, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:19:10,129 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T14:19:10,130 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731421150129"}]},"ts":"9223372036854775807"} 2024-11-12T14:19:10,133 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T14:19:10,133 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 33074d5f6b8c90fb791f1838297603c6, NAME => 'TestAcidGuarantees,,1731421119566.33074d5f6b8c90fb791f1838297603c6.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T14:19:10,133 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T14:19:10,133 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731421150133"}]},"ts":"9223372036854775807"} 2024-11-12T14:19:10,137 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T14:19:10,139 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=135, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:19:10,141 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 108 msec 2024-11-12T14:19:10,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-12T14:19:10,150 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T14:19:10,152 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:19:10,161 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testScanAtomicity Thread=246 (was 248), OpenFileDescriptor=449 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=833 (was 858), ProcessCount=11 (was 11), AvailableMemoryMB=397 (was 1334) 2024-11-12T14:19:10,172 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=246, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=833, ProcessCount=11, AvailableMemoryMB=397 2024-11-12T14:19:10,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T14:19:10,174 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T14:19:10,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-12T14:19:10,177 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-12T14:19:10,177 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:19:10,177 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 136 2024-11-12T14:19:10,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-12T14:19:10,178 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-12T14:19:10,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742389_1565 (size=1061) 2024-11-12T14:19:10,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-12T14:19:10,335 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-12T14:19:10,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-12T14:19:10,597 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3 2024-11-12T14:19:10,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742390_1566 (size=53) 2024-11-12T14:19:10,605 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:19:10,605 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing e1aa16d140ed3f62a95c849e7bf6cc7a, disabling compactions & flushes 2024-11-12T14:19:10,605 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:10,605 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:10,605 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. after waiting 0 ms 2024-11-12T14:19:10,605 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:10,605 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:10,605 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for e1aa16d140ed3f62a95c849e7bf6cc7a: Waiting for close lock at 1731421150605Disabling compacts and flushes for region at 1731421150605Disabling writes for close at 1731421150605Writing region close event to WAL at 1731421150605Closed at 1731421150605 2024-11-12T14:19:10,606 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-12T14:19:10,607 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731421150606"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731421150606"}]},"ts":"1731421150606"} 2024-11-12T14:19:10,608 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-12T14:19:10,609 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-12T14:19:10,609 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731421150609"}]},"ts":"1731421150609"} 2024-11-12T14:19:10,610 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-12T14:19:10,610 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1aa16d140ed3f62a95c849e7bf6cc7a, ASSIGN}] 2024-11-12T14:19:10,611 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1aa16d140ed3f62a95c849e7bf6cc7a, ASSIGN 2024-11-12T14:19:10,612 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1aa16d140ed3f62a95c849e7bf6cc7a, ASSIGN; state=OFFLINE, location=2b6d221c5cde,42925,1731420990822; forceNewPlan=false, retain=false 2024-11-12T14:19:10,762 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=e1aa16d140ed3f62a95c849e7bf6cc7a, regionState=OPENING, regionLocation=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:10,764 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=137, ppid=136, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1aa16d140ed3f62a95c849e7bf6cc7a, ASSIGN because future has completed 2024-11-12T14:19:10,764 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822}] 2024-11-12T14:19:10,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-12T14:19:10,918 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:10,918 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7752): Opening region: {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:19:10,919 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:10,919 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:19:10,919 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7794): checking encryption for e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:10,919 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(7797): checking classloading for e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:10,920 INFO [StoreOpener-e1aa16d140ed3f62a95c849e7bf6cc7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:10,921 INFO [StoreOpener-e1aa16d140ed3f62a95c849e7bf6cc7a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T14:19:10,922 INFO [StoreOpener-e1aa16d140ed3f62a95c849e7bf6cc7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1aa16d140ed3f62a95c849e7bf6cc7a columnFamilyName A 2024-11-12T14:19:10,922 DEBUG [StoreOpener-e1aa16d140ed3f62a95c849e7bf6cc7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:19:10,924 INFO [StoreOpener-e1aa16d140ed3f62a95c849e7bf6cc7a-1 {}] regionserver.HStore(327): Store=e1aa16d140ed3f62a95c849e7bf6cc7a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:19:10,924 INFO [StoreOpener-e1aa16d140ed3f62a95c849e7bf6cc7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:10,925 INFO [StoreOpener-e1aa16d140ed3f62a95c849e7bf6cc7a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T14:19:10,926 INFO [StoreOpener-e1aa16d140ed3f62a95c849e7bf6cc7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1aa16d140ed3f62a95c849e7bf6cc7a columnFamilyName B 2024-11-12T14:19:10,926 DEBUG [StoreOpener-e1aa16d140ed3f62a95c849e7bf6cc7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:19:10,926 INFO [StoreOpener-e1aa16d140ed3f62a95c849e7bf6cc7a-1 {}] regionserver.HStore(327): Store=e1aa16d140ed3f62a95c849e7bf6cc7a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:19:10,926 INFO [StoreOpener-e1aa16d140ed3f62a95c849e7bf6cc7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:10,927 INFO [StoreOpener-e1aa16d140ed3f62a95c849e7bf6cc7a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T14:19:10,927 INFO [StoreOpener-e1aa16d140ed3f62a95c849e7bf6cc7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1aa16d140ed3f62a95c849e7bf6cc7a columnFamilyName C 2024-11-12T14:19:10,927 DEBUG [StoreOpener-e1aa16d140ed3f62a95c849e7bf6cc7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:19:10,928 INFO [StoreOpener-e1aa16d140ed3f62a95c849e7bf6cc7a-1 {}] regionserver.HStore(327): Store=e1aa16d140ed3f62a95c849e7bf6cc7a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:19:10,928 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:10,928 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1038): replaying wal for e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:10,929 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:10,929 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:10,930 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1048): stopping wal replay for e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:10,930 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1060): Cleaning up temporary data for e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:10,932 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T14:19:10,933 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1093): writing seq id for e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:10,935 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-12T14:19:10,935 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1114): Opened e1aa16d140ed3f62a95c849e7bf6cc7a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=73798882, jitterRate=0.09968903660774231}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T14:19:10,935 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:10,936 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegion(1006): Region open journal for e1aa16d140ed3f62a95c849e7bf6cc7a: Running coprocessor pre-open hook at 1731421150919Writing region info on filesystem at 1731421150919Initializing all the Stores at 1731421150919Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731421150920 (+1 ms)Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731421150920Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731421150920Cleaning up temporary data from old regions at 1731421150930 (+10 ms)Running coprocessor post-open hooks at 1731421150935 (+5 ms)Region opened successfully at 1731421150936 (+1 ms) 2024-11-12T14:19:10,937 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., pid=138, masterSystemTime=1731421150915 2024-11-12T14:19:10,939 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:10,939 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=138}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:10,939 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=137 updating hbase:meta row=e1aa16d140ed3f62a95c849e7bf6cc7a, regionState=OPEN, openSeqNum=2, regionLocation=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:10,941 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=138, ppid=137, state=RUNNABLE, hasLock=false; OpenRegionProcedure e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 because future has completed 2024-11-12T14:19:10,943 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-12T14:19:10,944 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; OpenRegionProcedure e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 in 178 msec 2024-11-12T14:19:10,945 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=137, resume processing ppid=136 2024-11-12T14:19:10,945 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, ppid=136, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1aa16d140ed3f62a95c849e7bf6cc7a, ASSIGN in 333 msec 2024-11-12T14:19:10,946 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-12T14:19:10,946 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731421150946"}]},"ts":"1731421150946"} 2024-11-12T14:19:10,948 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-12T14:19:10,949 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=136, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-12T14:19:10,951 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 775 msec 2024-11-12T14:19:11,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=136 2024-11-12T14:19:11,319 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:19:11,320 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@559e6aa9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:11,320 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:19:11,320 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:19:11,321 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:19:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:19:11,321 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:19:11,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@731ab9bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:11,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:19:11,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:19:11,322 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:11,322 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:19:11,323 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30a05118, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:11,323 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:19:11,324 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:19:11,324 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:19:11,326 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37606, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:19:11,326 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-12T14:19:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 2b6d221c5cde,45591,1731420989939 2024-11-12T14:19:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@67e98e3a 2024-11-12T14:19:11,327 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-12T14:19:11,328 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54688, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-12T14:19:11,329 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-12T14:19:11,330 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'EAGER', 'hbase.hregion.memstore.mslab.enabled' => 'false', 'hbase.memstore.inmemoryflush.threshold.factor' => '0.9', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-12T14:19:11,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-12T14:19:11,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742391_1567 (size=1097) 2024-11-12T14:19:11,744 DEBUG [PEWorker-2 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.1061 2024-11-12T14:19:11,744 INFO [PEWorker-2 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.1097 2024-11-12T14:19:11,746 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T14:19:11,747 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1aa16d140ed3f62a95c849e7bf6cc7a, REOPEN/MOVE}] 2024-11-12T14:19:11,748 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1aa16d140ed3f62a95c849e7bf6cc7a, REOPEN/MOVE 2024-11-12T14:19:11,749 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=e1aa16d140ed3f62a95c849e7bf6cc7a, regionState=CLOSING, regionLocation=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:11,750 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1aa16d140ed3f62a95c849e7bf6cc7a, REOPEN/MOVE because future has completed 2024-11-12T14:19:11,750 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T14:19:11,750 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822}] 2024-11-12T14:19:11,902 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:11,902 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T14:19:11,903 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing e1aa16d140ed3f62a95c849e7bf6cc7a, disabling compactions & flushes 2024-11-12T14:19:11,903 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:11,903 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:11,903 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. after waiting 0 ms 2024-11-12T14:19:11,903 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:11,918 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-12T14:19:11,918 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:11,918 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for e1aa16d140ed3f62a95c849e7bf6cc7a: Waiting for close lock at 1731421151903Running coprocessor pre-close hooks at 1731421151903Disabling compacts and flushes for region at 1731421151903Disabling writes for close at 1731421151903Writing region close event to WAL at 1731421151906 (+3 ms)Running coprocessor post-close hooks at 1731421151918 (+12 ms)Closed at 1731421151918 2024-11-12T14:19:11,918 WARN [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionServer(3299): Not adding moved region record: e1aa16d140ed3f62a95c849e7bf6cc7a to self. 2024-11-12T14:19:11,920 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:11,920 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=e1aa16d140ed3f62a95c849e7bf6cc7a, regionState=CLOSED 2024-11-12T14:19:11,922 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 because future has completed 2024-11-12T14:19:11,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-12T14:19:11,924 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 in 172 msec 2024-11-12T14:19:11,924 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1aa16d140ed3f62a95c849e7bf6cc7a, REOPEN/MOVE; state=CLOSED, location=2b6d221c5cde,42925,1731420990822; forceNewPlan=false, retain=true 2024-11-12T14:19:12,075 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=e1aa16d140ed3f62a95c849e7bf6cc7a, regionState=OPENING, regionLocation=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:12,077 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1aa16d140ed3f62a95c849e7bf6cc7a, REOPEN/MOVE because future has completed 2024-11-12T14:19:12,077 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822}] 2024-11-12T14:19:12,232 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:12,232 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7752): Opening region: {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} 2024-11-12T14:19:12,233 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:12,233 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-12T14:19:12,233 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7794): checking encryption for e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:12,233 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(7797): checking classloading for e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:12,234 INFO [StoreOpener-e1aa16d140ed3f62a95c849e7bf6cc7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:12,234 INFO [StoreOpener-e1aa16d140ed3f62a95c849e7bf6cc7a-1 {}] regionserver.CompactingMemStore(122): Store=A, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T14:19:12,234 INFO [StoreOpener-e1aa16d140ed3f62a95c849e7bf6cc7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1aa16d140ed3f62a95c849e7bf6cc7a columnFamilyName A 2024-11-12T14:19:12,235 DEBUG [StoreOpener-e1aa16d140ed3f62a95c849e7bf6cc7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:19:12,236 INFO [StoreOpener-e1aa16d140ed3f62a95c849e7bf6cc7a-1 {}] regionserver.HStore(327): Store=e1aa16d140ed3f62a95c849e7bf6cc7a/A, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:19:12,236 INFO [StoreOpener-e1aa16d140ed3f62a95c849e7bf6cc7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:12,236 INFO [StoreOpener-e1aa16d140ed3f62a95c849e7bf6cc7a-1 {}] regionserver.CompactingMemStore(122): Store=B, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T14:19:12,237 INFO [StoreOpener-e1aa16d140ed3f62a95c849e7bf6cc7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1aa16d140ed3f62a95c849e7bf6cc7a columnFamilyName B 2024-11-12T14:19:12,237 DEBUG [StoreOpener-e1aa16d140ed3f62a95c849e7bf6cc7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:19:12,237 INFO [StoreOpener-e1aa16d140ed3f62a95c849e7bf6cc7a-1 {}] regionserver.HStore(327): Store=e1aa16d140ed3f62a95c849e7bf6cc7a/B, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:19:12,237 INFO [StoreOpener-e1aa16d140ed3f62a95c849e7bf6cc7a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:12,237 INFO [StoreOpener-e1aa16d140ed3f62a95c849e7bf6cc7a-1 {}] regionserver.CompactingMemStore(122): Store=C, in-memory flush size threshold=38.40 KB, immutable segments index type=ARRAY_MAP, compactor=EAGER, pipelineThreshold=2, compactionCellMax=10 2024-11-12T14:19:12,238 INFO [StoreOpener-e1aa16d140ed3f62a95c849e7bf6cc7a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region e1aa16d140ed3f62a95c849e7bf6cc7a columnFamilyName C 2024-11-12T14:19:12,238 DEBUG [StoreOpener-e1aa16d140ed3f62a95c849e7bf6cc7a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:19:12,238 INFO [StoreOpener-e1aa16d140ed3f62a95c849e7bf6cc7a-1 {}] regionserver.HStore(327): Store=e1aa16d140ed3f62a95c849e7bf6cc7a/C, memstore type=CompactingMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-12T14:19:12,238 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1212): Setting FlushNonSloppyStoresFirstPolicy for the region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:12,238 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1038): replaying wal for e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:12,238 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:12,239 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:12,240 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1048): stopping wal replay for e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:12,240 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1060): Cleaning up temporary data for e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:12,240 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-12T14:19:12,241 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1093): writing seq id for e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:12,242 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1114): Opened e1aa16d140ed3f62a95c849e7bf6cc7a; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60951482, jitterRate=-0.09175214171409607}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-12T14:19:12,242 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1122): Running coprocessor post-open hooks for e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:12,242 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegion(1006): Region open journal for e1aa16d140ed3f62a95c849e7bf6cc7a: Running coprocessor pre-open hook at 1731421152233Writing region info on filesystem at 1731421152233Initializing all the Stores at 1731421152233Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731421152233Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731421152234 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731421152234Cleaning up temporary data from old regions at 1731421152240 (+6 ms)Running coprocessor post-open hooks at 1731421152242 (+2 ms)Region opened successfully at 1731421152242 2024-11-12T14:19:12,243 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., pid=143, masterSystemTime=1731421152230 2024-11-12T14:19:12,244 DEBUG [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:12,244 INFO [RS_OPEN_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_OPEN_REGION, pid=143}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:12,244 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=e1aa16d140ed3f62a95c849e7bf6cc7a, regionState=OPEN, openSeqNum=5, regionLocation=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:12,246 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=141, state=RUNNABLE, hasLock=false; OpenRegionProcedure e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 because future has completed 2024-11-12T14:19:12,248 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=141 2024-11-12T14:19:12,248 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=141, state=SUCCESS, hasLock=false; OpenRegionProcedure e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 in 169 msec 2024-11-12T14:19:12,249 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-12T14:19:12,249 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1aa16d140ed3f62a95c849e7bf6cc7a, REOPEN/MOVE in 501 msec 2024-11-12T14:19:12,250 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-12T14:19:12,250 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 503 msec 2024-11-12T14:19:12,252 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 921 msec 2024-11-12T14:19:12,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-12T14:19:12,252 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:19:12,253 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8257879, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:12,253 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:19:12,253 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:19:12,254 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:19:12,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:19:12,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:19:12,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d24e734, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:12,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:19:12,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:19:12,254 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:12,255 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40664, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:19:12,255 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@646e7c3d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:12,256 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4a3fe0ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:12,256 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:19:12,256 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:19:12,256 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:19:12,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:19:12,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:19:12,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7f81c8a7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:12,256 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:19:12,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:19:12,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:12,257 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40678, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:19:12,257 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@442ec98f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:12,258 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9e20296, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:12,258 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:19:12,258 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:19:12,258 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:19:12,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:19:12,258 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:19:12,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d4ddb9f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:12,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:19:12,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:19:12,259 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:12,259 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40690, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:19:12,260 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4952855, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:12,260 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c56a33e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:12,260 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:19:12,260 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:19:12,261 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:19:12,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:19:12,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:19:12,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c5b75b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:12,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:19:12,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:19:12,261 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:12,262 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40696, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:19:12,262 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71ee0781, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:12,262 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@362a517d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:12,262 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:19:12,263 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:19:12,263 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:19:12,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:19:12,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:19:12,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34029c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:12,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:19:12,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:19:12,263 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:12,264 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40708, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:19:12,264 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c01b8bc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:12,265 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@732b7d63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:12,265 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:19:12,265 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:19:12,265 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:19:12,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:19:12,265 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:19:12,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@141673ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:12,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:19:12,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:19:12,266 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:12,266 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40734, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:19:12,267 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@304165ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:12,267 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76841744, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:12,267 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:19:12,267 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:19:12,268 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:19:12,268 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:19:12,268 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:19:12,268 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f595c2f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:12,268 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:19:12,268 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:19:12,268 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:12,269 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40760, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:19:12,269 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@183d684e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:12,270 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78ab0eab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:12,270 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:19:12,270 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:19:12,270 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:19:12,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:19:12,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:19:12,270 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c349edc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:12,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:19:12,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:19:12,271 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:12,271 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40772, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:19:12,272 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@9c3e51a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:12,272 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7b86f115, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:12,272 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:19:12,272 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:19:12,273 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:19:12,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:19:12,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:19:12,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46579039, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:12,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:19:12,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:19:12,273 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:12,274 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40794, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:19:12,274 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e4d8aed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:12,276 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fe4550b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:12,276 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 2b6d221c5cde,45591,-1 for getting cluster id 2024-11-12T14:19:12,276 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-12T14:19:12,277 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = 'c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a' 2024-11-12T14:19:12,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-12T14:19:12,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "c6cc892a-4d7b-49d4-9809-2cacf2fb4c4a" 2024-11-12T14:19:12,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@51178f5c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:12,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [2b6d221c5cde,45591,-1] 2024-11-12T14:19:12,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-12T14:19:12,277 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:12,278 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:40820, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-12T14:19:12,278 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5284e269, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-12T14:19:12,279 DEBUG [Thread-2541 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:19:12,279 DEBUG [Thread-2526 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:19:12,279 DEBUG [Thread-2528 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:19:12,279 DEBUG [Thread-2543 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:19:12,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:19:12,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:19:12,279 DEBUG [Thread-2524 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:19:12,279 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:19:12,279 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:19:12,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:19:12,280 DEBUG [Thread-2530 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:19:12,280 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:19:12,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:19:12,280 DEBUG [Thread-2535 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:19:12,280 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:19:12,280 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:19:12,281 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43748, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:19:12,281 DEBUG [Thread-2532 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:19:12,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:19:12,281 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:19:12,281 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43764, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:19:12,281 DEBUG [Thread-2537 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:19:12,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees 2024-11-12T14:19:12,281 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:19:12,282 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:19:12,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:19:12,282 DEBUG [Thread-2539 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-12T14:19:12,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:19:12,282 DEBUG [Thread-2537 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:19:12,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-12T14:19:12,282 DEBUG [Thread-2537 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:19:12,282 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:19:12,283 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:19:12,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=2b6d221c5cde,42925,1731420990822, seqNum=-1] 2024-11-12T14:19:12,283 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-12T14:19:12,283 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43766, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:19:12,283 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43776, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:19:12,284 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43770, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:19:12,284 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5] 2024-11-12T14:19:12,284 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43800, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:19:12,284 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43768, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:19:12,284 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43790, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:19:12,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5] 2024-11-12T14:19:12,284 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:19:12,285 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43802, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:19:12,285 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43816, version=3.0.0-beta-2-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-12T14:19:12,285 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5] 2024-11-12T14:19:12,285 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=144, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:19:12,285 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:19:12,286 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5] 2024-11-12T14:19:12,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5] 2024-11-12T14:19:12,287 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5] 2024-11-12T14:19:12,288 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5] 2024-11-12T14:19:12,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5] 2024-11-12T14:19:12,289 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5] 2024-11-12T14:19:12,289 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5] 2024-11-12T14:19:12,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:12,292 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1aa16d140ed3f62a95c849e7bf6cc7a 3/3 column families, dataSize=46.96 KB heapSize=140.20 KB 2024-11-12T14:19:12,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=A 2024-11-12T14:19:12,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:12,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=B 2024-11-12T14:19:12,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:12,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=C 2024-11-12T14:19:12,292 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:12,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-6 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731421152299, max timestamp=1731421152326Num uniques 859; 2024-11-12T14:19:12,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-2 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731421152299, max timestamp=1731421152326Num uniques 859; 2024-11-12T14:19:12,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-6 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-12T14:19:12,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-2 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-12T14:19:12,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-3 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=type=CellArrayImmutableSegment, empty=no, cellCount=359, cellSize=16435, totalHeapSize=37576, min timestamp=1731421152299, max timestamp=1731421152326Num uniques 859; 2024-11-12T14:19:12,329 DEBUG [RS_IN_MEMORY_COMPACTION-regionserver/2b6d221c5cde:0-3 {}] regionserver.CompactionPipeline(163): Suffix data size=39335, new segment data size=16435, suffix heap size=117064,new segment heap size=37576 　suffix off heap size=0, new segment off heap size=0, suffix cells count=859, new segment cells count=359 2024-11-12T14:19:12,332 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c20f5ec30ea54a56a034b880814142e6_e1aa16d140ed3f62a95c849e7bf6cc7a is 50, key is test_row_0/A:col10/1731421152290/Put/seqid=0 2024-11-12T14:19:12,352 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421212343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421212344, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,353 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:12,353 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421212345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:12,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421212346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:12,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,354 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:12,354 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:12,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:12,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421212352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:12,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,355 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:12,373 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742392_1568 (size=14594) 2024-11-12T14:19:12,374 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:19:12,377 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c20f5ec30ea54a56a034b880814142e6_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c20f5ec30ea54a56a034b880814142e6_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:12,378 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/8455b70638034900aac82439d633619f, store: [table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:12,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/8455b70638034900aac82439d633619f is 175, key is test_row_0/A:col10/1731421152290/Put/seqid=0 2024-11-12T14:19:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-12T14:19:12,405 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742393_1569 (size=39549) 2024-11-12T14:19:12,406 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=14, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/8455b70638034900aac82439d633619f 2024-11-12T14:19:12,439 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-12T14:19:12,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:12,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:12,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:12,439 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:12,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:12,450 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/35ca1eddfda44f5c94a6e4ecd140ca67 is 50, key is test_row_0/B:col10/1731421152290/Put/seqid=0 2024-11-12T14:19:12,462 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421212459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:12,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421212459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:12,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:12,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421212459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:12,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,463 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:12,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,463 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:12,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:12,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421212460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:12,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,463 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:12,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:12,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421212460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:12,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,464 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:12,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,464 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:12,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742394_1570 (size=12001) 2024-11-12T14:19:12,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/35ca1eddfda44f5c94a6e4ecd140ca67 2024-11-12T14:19:12,514 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/17b12a50b8294ab8b7339fd5fa6eb5f1 is 50, key is test_row_0/C:col10/1731421152290/Put/seqid=0 2024-11-12T14:19:12,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742395_1571 (size=12001) 2024-11-12T14:19:12,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-12T14:19:12,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:12,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:12,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:12,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:12,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:12,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-12T14:19:12,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421212670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:12,672 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:12,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421212671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:12,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421212671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:12,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421212671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:12,673 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,674 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:12,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:12,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421212671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:12,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:12,747 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-12T14:19:12,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:12,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:12,747 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:12,747 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:12,748 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:12,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:12,900 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-12T14:19:12,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:12,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:12,900 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:12,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] handler.RSProcedureHandler(58): pid=145 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:12,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=145 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:12,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=145 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:12,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-12T14:19:12,963 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/17b12a50b8294ab8b7339fd5fa6eb5f1 2024-11-12T14:19:12,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/8455b70638034900aac82439d633619f as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/8455b70638034900aac82439d633619f 2024-11-12T14:19:12,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/8455b70638034900aac82439d633619f, entries=200, sequenceid=14, filesize=38.6 K 2024-11-12T14:19:12,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/35ca1eddfda44f5c94a6e4ecd140ca67 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/35ca1eddfda44f5c94a6e4ecd140ca67 2024-11-12T14:19:12,977 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/35ca1eddfda44f5c94a6e4ecd140ca67, entries=150, sequenceid=14, filesize=11.7 K 2024-11-12T14:19:12,978 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/17b12a50b8294ab8b7339fd5fa6eb5f1 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/17b12a50b8294ab8b7339fd5fa6eb5f1 2024-11-12T14:19:12,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:12,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421212980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:12,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:12,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,981 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:12,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421212980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:12,981 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:12,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:12,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421212980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:12,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:12,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:12,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421212980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:12,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:12,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:12,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421212981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:12,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:12,983 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:12,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/17b12a50b8294ab8b7339fd5fa6eb5f1, entries=150, sequenceid=14, filesize=11.7 K 2024-11-12T14:19:12,995 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~140.16 KB/143520, currentSize=140.89 KB/144270 for e1aa16d140ed3f62a95c849e7bf6cc7a in 703ms, sequenceid=14, compaction requested=false 2024-11-12T14:19:12,995 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:13,053 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=145 2024-11-12T14:19:13,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:13,053 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2902): Flushing e1aa16d140ed3f62a95c849e7bf6cc7a 3/3 column families, dataSize=140.89 KB heapSize=386.16 KB 2024-11-12T14:19:13,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=A 2024-11-12T14:19:13,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:19:13,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=B 2024-11-12T14:19:13,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:19:13,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=C 2024-11-12T14:19:13,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:19:13,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411123491b1b36ac44789af5bea616ff1d24e_e1aa16d140ed3f62a95c849e7bf6cc7a is 50, key is test_row_0/A:col10/1731421152350/Put/seqid=0 2024-11-12T14:19:13,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742396_1572 (size=12154) 2024-11-12T14:19:13,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-12T14:19:13,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:19:13,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:13,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:13,495 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411123491b1b36ac44789af5bea616ff1d24e_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123491b1b36ac44789af5bea616ff1d24e_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:13,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/04e2e9800ac44fa880d2e689cbdec5d5, store: [table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:13,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/04e2e9800ac44fa880d2e689cbdec5d5 is 175, key is test_row_0/A:col10/1731421152350/Put/seqid=0 2024-11-12T14:19:13,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:13,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421213500, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:13,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:13,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,510 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:13,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421213501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:13,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,510 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:13,514 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:13,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421213506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,516 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:13,519 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:13,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421213512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:13,520 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421213513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,520 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:13,525 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742397_1573 (size=30955) 2024-11-12T14:19:13,526 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=48, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/04e2e9800ac44fa880d2e689cbdec5d5 2024-11-12T14:19:13,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/640e88ef02ed4aa5a52fac019048b96d is 50, key is test_row_0/B:col10/1731421152350/Put/seqid=0 2024-11-12T14:19:13,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742398_1574 (size=12001) 2024-11-12T14:19:13,586 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/640e88ef02ed4aa5a52fac019048b96d 2024-11-12T14:19:13,614 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/7cd769140fb34b94aef8b4fca7e6c312 is 50, key is test_row_0/C:col10/1731421152350/Put/seqid=0 2024-11-12T14:19:13,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742399_1575 (size=12001) 2024-11-12T14:19:13,621 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=48 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/7cd769140fb34b94aef8b4fca7e6c312 2024-11-12T14:19:13,622 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:13,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421213619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:13,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:13,626 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:13,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421213619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,626 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:13,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421213619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:13,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:13,627 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/04e2e9800ac44fa880d2e689cbdec5d5 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/04e2e9800ac44fa880d2e689cbdec5d5 2024-11-12T14:19:13,631 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/04e2e9800ac44fa880d2e689cbdec5d5, entries=150, sequenceid=48, filesize=30.2 K 2024-11-12T14:19:13,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/640e88ef02ed4aa5a52fac019048b96d as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/640e88ef02ed4aa5a52fac019048b96d 2024-11-12T14:19:13,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:13,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421213630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:13,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:13,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421213630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:13,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,633 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:13,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:13,635 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/640e88ef02ed4aa5a52fac019048b96d, entries=150, sequenceid=48, filesize=11.7 K 2024-11-12T14:19:13,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/7cd769140fb34b94aef8b4fca7e6c312 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/7cd769140fb34b94aef8b4fca7e6c312 2024-11-12T14:19:13,638 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/7cd769140fb34b94aef8b4fca7e6c312, entries=150, sequenceid=48, filesize=11.7 K 2024-11-12T14:19:13,639 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~386.11 KB/395376, currentSize=53.67 KB/54960 for e1aa16d140ed3f62a95c849e7bf6cc7a in 586ms, sequenceid=48, compaction requested=false 2024-11-12T14:19:13,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.HRegion(2603): Flush status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:13,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:13,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=145}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=145 2024-11-12T14:19:13,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=145 2024-11-12T14:19:13,642 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-12T14:19:13,642 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3560 sec 2024-11-12T14:19:13,644 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=144, table=TestAcidGuarantees in 1.3630 sec 2024-11-12T14:19:13,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:13,832 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1aa16d140ed3f62a95c849e7bf6cc7a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T14:19:13,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=A 2024-11-12T14:19:13,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:13,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=B 2024-11-12T14:19:13,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:13,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=C 2024-11-12T14:19:13,833 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:13,840 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d2f8d20a2c6d471b86c1f9215e79ba9f_e1aa16d140ed3f62a95c849e7bf6cc7a is 50, key is test_row_0/A:col10/1731421153509/Put/seqid=0 2024-11-12T14:19:13,857 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:13,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421213848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,857 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:13,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:13,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421213854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,860 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,860 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:13,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421213854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:13,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:13,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421213855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:13,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:13,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421213857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:13,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,864 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:13,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742400_1576 (size=14594) 2024-11-12T14:19:13,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:13,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421213970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:13,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:13,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,974 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:13,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421213970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:13,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:13,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421213970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:13,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:13,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421213970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:13,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421213971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:13,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:13,977 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:14,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:14,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421214179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:14,181 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:14,181 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:14,182 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:14,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421214179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:14,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:14,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:14,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:14,182 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:14,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421214179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:14,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:14,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:14,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:14,182 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:14,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421214179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:14,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:14,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:14,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:14,182 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:14,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421214179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:14,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:14,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:14,183 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:14,280 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:19:14,282 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112d2f8d20a2c6d471b86c1f9215e79ba9f_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d2f8d20a2c6d471b86c1f9215e79ba9f_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:14,283 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/be3c126d3bca491d86acc896c21fb04a, store: [table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:14,283 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/be3c126d3bca491d86acc896c21fb04a is 175, key is test_row_0/A:col10/1731421153509/Put/seqid=0 2024-11-12T14:19:14,296 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742401_1577 (size=39549) 2024-11-12T14:19:14,296 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=62, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/be3c126d3bca491d86acc896c21fb04a 2024-11-12T14:19:14,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/4836afae9c0c47209581d5af758eaa70 is 50, key is test_row_0/B:col10/1731421153509/Put/seqid=0 2024-11-12T14:19:14,310 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742402_1578 (size=12001) 2024-11-12T14:19:14,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-12T14:19:14,430 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:19:14,432 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:19:14,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees 2024-11-12T14:19:14,439 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:19:14,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-12T14:19:14,440 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=146, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:19:14,441 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:19:14,496 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421214491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:14,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421214491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:14,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:14,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421214491, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:14,497 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:14,497 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:14,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:14,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421214492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:14,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:14,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421214492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:14,498 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:14,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-12T14:19:14,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-12T14:19:14,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:14,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:14,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:14,594 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] handler.RSProcedureHandler(58): pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:14,594 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=147 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:14,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=147 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:14,711 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/4836afae9c0c47209581d5af758eaa70 2024-11-12T14:19:14,717 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/916993ff540142148b84028c608eeec9 is 50, key is test_row_0/C:col10/1731421153509/Put/seqid=0 2024-11-12T14:19:14,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742403_1579 (size=12001) 2024-11-12T14:19:14,728 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=62 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/916993ff540142148b84028c608eeec9 2024-11-12T14:19:14,733 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/be3c126d3bca491d86acc896c21fb04a as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/be3c126d3bca491d86acc896c21fb04a 2024-11-12T14:19:14,735 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/be3c126d3bca491d86acc896c21fb04a, entries=200, sequenceid=62, filesize=38.6 K 2024-11-12T14:19:14,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/4836afae9c0c47209581d5af758eaa70 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/4836afae9c0c47209581d5af758eaa70 2024-11-12T14:19:14,740 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/4836afae9c0c47209581d5af758eaa70, entries=150, sequenceid=62, filesize=11.7 K 2024-11-12T14:19:14,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/916993ff540142148b84028c608eeec9 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/916993ff540142148b84028c608eeec9 2024-11-12T14:19:14,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/916993ff540142148b84028c608eeec9, entries=150, sequenceid=62, filesize=11.7 K 2024-11-12T14:19:14,745 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e1aa16d140ed3f62a95c849e7bf6cc7a in 912ms, sequenceid=62, compaction requested=true 2024-11-12T14:19:14,745 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:14,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1aa16d140ed3f62a95c849e7bf6cc7a:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:19:14,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:14,745 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:19:14,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1aa16d140ed3f62a95c849e7bf6cc7a:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:19:14,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:19:14,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1aa16d140ed3f62a95c849e7bf6cc7a:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:19:14,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T14:19:14,745 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:19:14,746 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110053 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:19:14,746 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): e1aa16d140ed3f62a95c849e7bf6cc7a/A is initiating minor compaction (all files) 2024-11-12T14:19:14,746 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1aa16d140ed3f62a95c849e7bf6cc7a/A in TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:14,746 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=147 2024-11-12T14:19:14,746 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/8455b70638034900aac82439d633619f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/04e2e9800ac44fa880d2e689cbdec5d5, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/be3c126d3bca491d86acc896c21fb04a] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp, totalSize=107.5 K 2024-11-12T14:19:14,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:14,746 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:14,746 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. files: [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/8455b70638034900aac82439d633619f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/04e2e9800ac44fa880d2e689cbdec5d5, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/be3c126d3bca491d86acc896c21fb04a] 2024-11-12T14:19:14,746 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2902): Flushing e1aa16d140ed3f62a95c849e7bf6cc7a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T14:19:14,746 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:19:14,746 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): e1aa16d140ed3f62a95c849e7bf6cc7a/B is initiating minor compaction (all files) 2024-11-12T14:19:14,746 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1aa16d140ed3f62a95c849e7bf6cc7a/B in TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:14,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=A 2024-11-12T14:19:14,746 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/35ca1eddfda44f5c94a6e4ecd140ca67, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/640e88ef02ed4aa5a52fac019048b96d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/4836afae9c0c47209581d5af758eaa70] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp, totalSize=35.2 K 2024-11-12T14:19:14,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:14,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=B 2024-11-12T14:19:14,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:14,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=C 2024-11-12T14:19:14,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:14,747 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 8455b70638034900aac82439d633619f, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731421152288 2024-11-12T14:19:14,747 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35ca1eddfda44f5c94a6e4ecd140ca67, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731421152288 2024-11-12T14:19:14,747 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 04e2e9800ac44fa880d2e689cbdec5d5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731421152339 2024-11-12T14:19:14,747 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 640e88ef02ed4aa5a52fac019048b96d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731421152339 2024-11-12T14:19:14,748 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4836afae9c0c47209581d5af758eaa70, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731421153509 2024-11-12T14:19:14,748 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting be3c126d3bca491d86acc896c21fb04a, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731421153509 2024-11-12T14:19:14,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c41b0842b27c42dca2cf1f1f200f4622_e1aa16d140ed3f62a95c849e7bf6cc7a is 50, key is test_row_0/A:col10/1731421153850/Put/seqid=0 2024-11-12T14:19:14,758 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1aa16d140ed3f62a95c849e7bf6cc7a#B#compaction#496 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:14,759 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/c292014e9b274a73a1df7034b707c602 is 50, key is test_row_0/B:col10/1731421153509/Put/seqid=0 2024-11-12T14:19:14,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-12T14:19:14,760 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:14,772 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112bc4bd88d6135447c9a0628099d6689da_e1aa16d140ed3f62a95c849e7bf6cc7a store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:14,774 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112bc4bd88d6135447c9a0628099d6689da_e1aa16d140ed3f62a95c849e7bf6cc7a, store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:14,775 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112bc4bd88d6135447c9a0628099d6689da_e1aa16d140ed3f62a95c849e7bf6cc7a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:14,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742404_1580 (size=12154) 2024-11-12T14:19:14,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742405_1581 (size=12104) 2024-11-12T14:19:14,828 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742406_1582 (size=4469) 2024-11-12T14:19:14,955 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-12T14:19:15,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:15,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:15,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:15,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421215016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:15,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:15,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,022 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:15,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421215016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:15,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:15,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:15,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421215020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:15,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,022 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:15,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:15,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421215021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:15,025 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:15,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421215021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:15,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,026 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:15,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,026 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:15,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-12T14:19:15,133 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:15,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421215129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:15,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:15,134 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:15,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421215129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:15,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:15,134 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:15,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421215129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:15,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,134 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:15,134 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:15,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421215129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:15,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,135 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:15,135 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:15,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421215129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:15,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,135 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:15,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:19:15,193 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c41b0842b27c42dca2cf1f1f200f4622_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c41b0842b27c42dca2cf1f1f200f4622_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:15,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/2244a0ffd17044fc91a42ec28d73a96f, store: [table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:15,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/2244a0ffd17044fc91a42ec28d73a96f is 175, key is test_row_0/A:col10/1731421153850/Put/seqid=0 2024-11-12T14:19:15,214 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/c292014e9b274a73a1df7034b707c602 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/c292014e9b274a73a1df7034b707c602 2024-11-12T14:19:15,218 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1aa16d140ed3f62a95c849e7bf6cc7a/B of e1aa16d140ed3f62a95c849e7bf6cc7a into c292014e9b274a73a1df7034b707c602(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:19:15,218 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:15,218 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., storeName=e1aa16d140ed3f62a95c849e7bf6cc7a/B, priority=13, startTime=1731421154745; duration=0sec 2024-11-12T14:19:15,218 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:19:15,218 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1aa16d140ed3f62a95c849e7bf6cc7a:B 2024-11-12T14:19:15,218 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:19:15,219 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:19:15,219 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): e1aa16d140ed3f62a95c849e7bf6cc7a/C is initiating minor compaction (all files) 2024-11-12T14:19:15,219 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1aa16d140ed3f62a95c849e7bf6cc7a/C in TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:15,219 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/17b12a50b8294ab8b7339fd5fa6eb5f1, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/7cd769140fb34b94aef8b4fca7e6c312, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/916993ff540142148b84028c608eeec9] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp, totalSize=35.2 K 2024-11-12T14:19:15,220 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17b12a50b8294ab8b7339fd5fa6eb5f1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731421152288 2024-11-12T14:19:15,220 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7cd769140fb34b94aef8b4fca7e6c312, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=48, earliestPutTs=1731421152339 2024-11-12T14:19:15,220 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 916993ff540142148b84028c608eeec9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731421153509 2024-11-12T14:19:15,226 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1aa16d140ed3f62a95c849e7bf6cc7a#C#compaction#498 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:15,227 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/eec7522f0081416c99830e33d53f9f2e is 50, key is test_row_0/C:col10/1731421153509/Put/seqid=0 2024-11-12T14:19:15,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742407_1583 (size=30955) 2024-11-12T14:19:15,229 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=80, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/2244a0ffd17044fc91a42ec28d73a96f 2024-11-12T14:19:15,231 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1aa16d140ed3f62a95c849e7bf6cc7a#A#compaction#497 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:15,231 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/a806108ad99544e084bcb3278bbba054 is 175, key is test_row_0/A:col10/1731421153509/Put/seqid=0 2024-11-12T14:19:15,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/ea6774aa8886452f82ac99a16c962744 is 50, key is test_row_0/B:col10/1731421153850/Put/seqid=0 2024-11-12T14:19:15,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742408_1584 (size=12104) 2024-11-12T14:19:15,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742409_1585 (size=31058) 2024-11-12T14:19:15,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742410_1586 (size=12001) 2024-11-12T14:19:15,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:15,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421215339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:15,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,342 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:15,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421215339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:15,342 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:15,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:15,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,343 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421215339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:15,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:15,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421215339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:15,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,343 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:15,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:15,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,343 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:15,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421215339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:15,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:15,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-12T14:19:15,651 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:15,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421215649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:15,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,651 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:15,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421215649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:15,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:15,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421215649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:15,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:15,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,653 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:15,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421215650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:15,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:15,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,654 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:15,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421215650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:15,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:15,654 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:15,667 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/eec7522f0081416c99830e33d53f9f2e as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/eec7522f0081416c99830e33d53f9f2e 2024-11-12T14:19:15,675 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1aa16d140ed3f62a95c849e7bf6cc7a/C of e1aa16d140ed3f62a95c849e7bf6cc7a into eec7522f0081416c99830e33d53f9f2e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:19:15,675 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:15,675 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., storeName=e1aa16d140ed3f62a95c849e7bf6cc7a/C, priority=13, startTime=1731421154745; duration=0sec 2024-11-12T14:19:15,675 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:15,675 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1aa16d140ed3f62a95c849e7bf6cc7a:C 2024-11-12T14:19:15,681 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/ea6774aa8886452f82ac99a16c962744 2024-11-12T14:19:15,689 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/a806108ad99544e084bcb3278bbba054 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/a806108ad99544e084bcb3278bbba054 2024-11-12T14:19:15,693 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1aa16d140ed3f62a95c849e7bf6cc7a/A of e1aa16d140ed3f62a95c849e7bf6cc7a into a806108ad99544e084bcb3278bbba054(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:19:15,693 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:15,693 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., storeName=e1aa16d140ed3f62a95c849e7bf6cc7a/A, priority=13, startTime=1731421154745; duration=0sec 2024-11-12T14:19:15,693 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:15,693 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1aa16d140ed3f62a95c849e7bf6cc7a:A 2024-11-12T14:19:15,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/7f09ee89d76542cda81751a4dd12d91e is 50, key is test_row_0/C:col10/1731421153850/Put/seqid=0 2024-11-12T14:19:15,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742411_1587 (size=12001) 2024-11-12T14:19:15,715 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=80 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/7f09ee89d76542cda81751a4dd12d91e 2024-11-12T14:19:15,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/2244a0ffd17044fc91a42ec28d73a96f as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/2244a0ffd17044fc91a42ec28d73a96f 2024-11-12T14:19:15,729 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/2244a0ffd17044fc91a42ec28d73a96f, entries=150, sequenceid=80, filesize=30.2 K 2024-11-12T14:19:15,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/ea6774aa8886452f82ac99a16c962744 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/ea6774aa8886452f82ac99a16c962744 2024-11-12T14:19:15,735 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/ea6774aa8886452f82ac99a16c962744, entries=150, sequenceid=80, filesize=11.7 K 2024-11-12T14:19:15,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/7f09ee89d76542cda81751a4dd12d91e as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/7f09ee89d76542cda81751a4dd12d91e 2024-11-12T14:19:15,739 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/7f09ee89d76542cda81751a4dd12d91e, entries=150, sequenceid=80, filesize=11.7 K 2024-11-12T14:19:15,740 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e1aa16d140ed3f62a95c849e7bf6cc7a in 994ms, sequenceid=80, compaction requested=false 2024-11-12T14:19:15,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.HRegion(2603): Flush status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:15,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:15,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=147}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=147 2024-11-12T14:19:15,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=147 2024-11-12T14:19:15,746 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-12T14:19:15,746 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3010 sec 2024-11-12T14:19:15,749 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=146, table=TestAcidGuarantees in 1.3150 sec 2024-11-12T14:19:16,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:16,161 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1aa16d140ed3f62a95c849e7bf6cc7a 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T14:19:16,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=A 2024-11-12T14:19:16,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:16,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=B 2024-11-12T14:19:16,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:16,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=C 2024-11-12T14:19:16,162 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:16,169 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411129d4673713f6747c38cadf67452805a21_e1aa16d140ed3f62a95c849e7bf6cc7a is 50, key is test_row_0/A:col10/1731421155019/Put/seqid=0 2024-11-12T14:19:16,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742412_1588 (size=14594) 2024-11-12T14:19:16,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:16,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421216175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:16,180 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:16,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421216176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:16,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,181 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:16,181 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:16,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:16,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421216179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:16,184 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:19:16,185 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:16,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421216181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:16,186 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:16,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421216181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:16,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,187 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:16,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,192 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:16,193 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411129d4673713f6747c38cadf67452805a21_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129d4673713f6747c38cadf67452805a21_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:16,193 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/85547f1332d4440981510c878887b607, store: [table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:16,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/85547f1332d4440981510c878887b607 is 175, key is test_row_0/A:col10/1731421155019/Put/seqid=0 2024-11-12T14:19:16,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742413_1589 (size=39549) 2024-11-12T14:19:16,197 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=98, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/85547f1332d4440981510c878887b607 2024-11-12T14:19:16,202 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/31ad260bda2343af9f6dab8cca4a4299 is 50, key is test_row_0/B:col10/1731421155019/Put/seqid=0 2024-11-12T14:19:16,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742414_1590 (size=12001) 2024-11-12T14:19:16,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:16,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421216290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:16,290 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421216290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:16,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421216290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:16,291 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:16,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421216290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:16,291 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,292 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,293 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:16,302 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:16,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421216300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:16,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:16,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421216499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:16,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421216499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421216499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:16,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:16,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421216499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:16,501 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:16,512 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:16,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421216509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:16,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,513 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:16,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=146 2024-11-12T14:19:16,589 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:19:16,589 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:19:16,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees 2024-11-12T14:19:16,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-12T14:19:16,591 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:19:16,592 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=148, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:19:16,592 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:19:16,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/31ad260bda2343af9f6dab8cca4a4299 2024-11-12T14:19:16,609 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/ed22870873c143aa93a09ce000ee5c85 is 50, key is test_row_0/C:col10/1731421155019/Put/seqid=0 2024-11-12T14:19:16,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742415_1591 (size=12001) 2024-11-12T14:19:16,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=98 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/ed22870873c143aa93a09ce000ee5c85 2024-11-12T14:19:16,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/85547f1332d4440981510c878887b607 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/85547f1332d4440981510c878887b607 2024-11-12T14:19:16,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/85547f1332d4440981510c878887b607, entries=200, sequenceid=98, filesize=38.6 K 2024-11-12T14:19:16,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/31ad260bda2343af9f6dab8cca4a4299 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/31ad260bda2343af9f6dab8cca4a4299 2024-11-12T14:19:16,624 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/31ad260bda2343af9f6dab8cca4a4299, entries=150, sequenceid=98, filesize=11.7 K 2024-11-12T14:19:16,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/ed22870873c143aa93a09ce000ee5c85 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/ed22870873c143aa93a09ce000ee5c85 2024-11-12T14:19:16,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/ed22870873c143aa93a09ce000ee5c85, entries=150, sequenceid=98, filesize=11.7 K 2024-11-12T14:19:16,629 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=100.63 KB/103050 for e1aa16d140ed3f62a95c849e7bf6cc7a in 468ms, sequenceid=98, compaction requested=true 2024-11-12T14:19:16,629 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:16,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1aa16d140ed3f62a95c849e7bf6cc7a:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:19:16,629 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:16,630 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:19:16,630 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:19:16,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1aa16d140ed3f62a95c849e7bf6cc7a:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:19:16,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:16,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1aa16d140ed3f62a95c849e7bf6cc7a:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:19:16,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:19:16,630 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:19:16,631 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): e1aa16d140ed3f62a95c849e7bf6cc7a/B is initiating minor compaction (all files) 2024-11-12T14:19:16,631 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1aa16d140ed3f62a95c849e7bf6cc7a/B in TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:16,631 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/c292014e9b274a73a1df7034b707c602, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/ea6774aa8886452f82ac99a16c962744, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/31ad260bda2343af9f6dab8cca4a4299] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp, totalSize=35.3 K 2024-11-12T14:19:16,631 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101562 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:19:16,631 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): e1aa16d140ed3f62a95c849e7bf6cc7a/A is initiating minor compaction (all files) 2024-11-12T14:19:16,631 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1aa16d140ed3f62a95c849e7bf6cc7a/A in TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:16,631 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/a806108ad99544e084bcb3278bbba054, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/2244a0ffd17044fc91a42ec28d73a96f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/85547f1332d4440981510c878887b607] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp, totalSize=99.2 K 2024-11-12T14:19:16,631 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:16,631 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. files: [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/a806108ad99544e084bcb3278bbba054, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/2244a0ffd17044fc91a42ec28d73a96f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/85547f1332d4440981510c878887b607] 2024-11-12T14:19:16,631 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting c292014e9b274a73a1df7034b707c602, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731421153509 2024-11-12T14:19:16,632 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting a806108ad99544e084bcb3278bbba054, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731421153509 2024-11-12T14:19:16,632 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting ea6774aa8886452f82ac99a16c962744, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731421153850 2024-11-12T14:19:16,632 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2244a0ffd17044fc91a42ec28d73a96f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731421153850 2024-11-12T14:19:16,632 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 31ad260bda2343af9f6dab8cca4a4299, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731421155014 2024-11-12T14:19:16,632 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 85547f1332d4440981510c878887b607, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731421155014 2024-11-12T14:19:16,638 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:16,639 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1aa16d140ed3f62a95c849e7bf6cc7a#B#compaction#505 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:16,640 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/d59297916ed24b6ea3f63c2f014a9455 is 50, key is test_row_0/B:col10/1731421155019/Put/seqid=0 2024-11-12T14:19:16,642 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111215d6993622464753a29fc9d0bd773458_e1aa16d140ed3f62a95c849e7bf6cc7a store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:16,644 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111215d6993622464753a29fc9d0bd773458_e1aa16d140ed3f62a95c849e7bf6cc7a, store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:16,644 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111215d6993622464753a29fc9d0bd773458_e1aa16d140ed3f62a95c849e7bf6cc7a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:16,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742416_1592 (size=12207) 2024-11-12T14:19:16,682 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/d59297916ed24b6ea3f63c2f014a9455 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/d59297916ed24b6ea3f63c2f014a9455 2024-11-12T14:19:16,695 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1aa16d140ed3f62a95c849e7bf6cc7a/B of e1aa16d140ed3f62a95c849e7bf6cc7a into d59297916ed24b6ea3f63c2f014a9455(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:19:16,695 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:16,695 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., storeName=e1aa16d140ed3f62a95c849e7bf6cc7a/B, priority=13, startTime=1731421156629; duration=0sec 2024-11-12T14:19:16,695 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:19:16,695 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1aa16d140ed3f62a95c849e7bf6cc7a:B 2024-11-12T14:19:16,695 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:19:16,696 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:19:16,696 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): e1aa16d140ed3f62a95c849e7bf6cc7a/C is initiating minor compaction (all files) 2024-11-12T14:19:16,696 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1aa16d140ed3f62a95c849e7bf6cc7a/C in TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:16,696 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/eec7522f0081416c99830e33d53f9f2e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/7f09ee89d76542cda81751a4dd12d91e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/ed22870873c143aa93a09ce000ee5c85] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp, totalSize=35.3 K 2024-11-12T14:19:16,696 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting eec7522f0081416c99830e33d53f9f2e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=62, earliestPutTs=1731421153509 2024-11-12T14:19:16,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742417_1593 (size=4469) 2024-11-12T14:19:16,697 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f09ee89d76542cda81751a4dd12d91e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=80, earliestPutTs=1731421153850 2024-11-12T14:19:16,697 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting ed22870873c143aa93a09ce000ee5c85, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731421155014 2024-11-12T14:19:16,700 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1aa16d140ed3f62a95c849e7bf6cc7a#A#compaction#504 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:16,700 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/f5949ebfcbee4e5687266bf370b5239e is 175, key is test_row_0/A:col10/1731421155019/Put/seqid=0 2024-11-12T14:19:16,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-12T14:19:16,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742418_1594 (size=31161) 2024-11-12T14:19:16,715 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1aa16d140ed3f62a95c849e7bf6cc7a#C#compaction#506 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:16,715 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/03f2be16f5f843929d8b25d276f94d88 is 50, key is test_row_0/C:col10/1731421155019/Put/seqid=0 2024-11-12T14:19:16,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742419_1595 (size=12207) 2024-11-12T14:19:16,734 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/03f2be16f5f843929d8b25d276f94d88 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/03f2be16f5f843929d8b25d276f94d88 2024-11-12T14:19:16,738 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1aa16d140ed3f62a95c849e7bf6cc7a/C of e1aa16d140ed3f62a95c849e7bf6cc7a into 03f2be16f5f843929d8b25d276f94d88(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:19:16,738 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:16,738 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., storeName=e1aa16d140ed3f62a95c849e7bf6cc7a/C, priority=13, startTime=1731421156630; duration=0sec 2024-11-12T14:19:16,738 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:16,738 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1aa16d140ed3f62a95c849e7bf6cc7a:C 2024-11-12T14:19:16,744 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=149 2024-11-12T14:19:16,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:16,744 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2902): Flushing e1aa16d140ed3f62a95c849e7bf6cc7a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T14:19:16,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=A 2024-11-12T14:19:16,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:16,744 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=B 2024-11-12T14:19:16,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:16,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=C 2024-11-12T14:19:16,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:16,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111210e2d5edf8d44eaba84a43e103bf45e7_e1aa16d140ed3f62a95c849e7bf6cc7a is 50, key is test_row_0/A:col10/1731421156179/Put/seqid=0 2024-11-12T14:19:16,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742420_1596 (size=12154) 2024-11-12T14:19:16,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:19:16,777 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111210e2d5edf8d44eaba84a43e103bf45e7_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111210e2d5edf8d44eaba84a43e103bf45e7_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:16,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/cba804fdea45429ab0b8658d33065559, store: [table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:16,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/cba804fdea45429ab0b8658d33065559 is 175, key is test_row_0/A:col10/1731421156179/Put/seqid=0 2024-11-12T14:19:16,782 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742421_1597 (size=30955) 2024-11-12T14:19:16,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:16,812 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:16,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:16,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421216820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:16,825 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:16,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421216821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:16,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,825 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:16,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:16,829 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:16,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:16,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421216825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:16,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421216825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:16,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:16,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,830 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:16,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:16,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421216826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:16,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,830 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:16,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-12T14:19:16,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:16,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421216929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:16,931 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,931 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,931 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:16,931 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:16,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421216930, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:16,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,931 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:16,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421216939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:16,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:16,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,942 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421216939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:16,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,942 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:16,942 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:16,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421216939, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:16,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:16,943 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:17,118 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/f5949ebfcbee4e5687266bf370b5239e as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/f5949ebfcbee4e5687266bf370b5239e 2024-11-12T14:19:17,124 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1aa16d140ed3f62a95c849e7bf6cc7a/A of e1aa16d140ed3f62a95c849e7bf6cc7a into f5949ebfcbee4e5687266bf370b5239e(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:19:17,124 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:17,124 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., storeName=e1aa16d140ed3f62a95c849e7bf6cc7a/A, priority=13, startTime=1731421156629; duration=0sec 2024-11-12T14:19:17,125 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:17,125 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1aa16d140ed3f62a95c849e7bf6cc7a:A 2024-11-12T14:19:17,140 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:17,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421217139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:17,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:17,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:17,141 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:17,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:17,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421217139, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:17,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:17,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:17,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:17,153 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:17,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421217150, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:17,153 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:17,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:17,154 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:17,154 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:17,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421217152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:17,155 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:17,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421217153, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:17,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:17,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:17,156 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:17,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:17,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:17,156 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:17,182 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/cba804fdea45429ab0b8658d33065559 2024-11-12T14:19:17,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/09385d7a5b20481cb75c65e4271772d5 is 50, key is test_row_0/B:col10/1731421156179/Put/seqid=0 2024-11-12T14:19:17,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742422_1598 (size=12001) 2024-11-12T14:19:17,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-12T14:19:17,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:17,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421217450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:17,452 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:17,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:17,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:17,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421217450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:17,452 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:17,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:17,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:17,453 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:17,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:17,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421217459, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:17,461 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:17,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:17,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:17,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421217469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:17,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421217469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:17,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:17,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:17,472 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:17,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:17,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:17,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:17,600 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/09385d7a5b20481cb75c65e4271772d5 2024-11-12T14:19:17,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/7a7a04b7327f4ea880f837a30e43e33b is 50, key is test_row_0/C:col10/1731421156179/Put/seqid=0 2024-11-12T14:19:17,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742423_1599 (size=12001) 2024-11-12T14:19:17,610 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/7a7a04b7327f4ea880f837a30e43e33b 2024-11-12T14:19:17,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/cba804fdea45429ab0b8658d33065559 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/cba804fdea45429ab0b8658d33065559 2024-11-12T14:19:17,620 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/cba804fdea45429ab0b8658d33065559, entries=150, sequenceid=118, filesize=30.2 K 2024-11-12T14:19:17,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/09385d7a5b20481cb75c65e4271772d5 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/09385d7a5b20481cb75c65e4271772d5 2024-11-12T14:19:17,623 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/09385d7a5b20481cb75c65e4271772d5, entries=150, sequenceid=118, filesize=11.7 K 2024-11-12T14:19:17,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/7a7a04b7327f4ea880f837a30e43e33b as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/7a7a04b7327f4ea880f837a30e43e33b 2024-11-12T14:19:17,635 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/7a7a04b7327f4ea880f837a30e43e33b, entries=150, sequenceid=118, filesize=11.7 K 2024-11-12T14:19:17,636 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=80.51 KB/82440 for e1aa16d140ed3f62a95c849e7bf6cc7a in 892ms, sequenceid=118, compaction requested=false 2024-11-12T14:19:17,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.HRegion(2603): Flush status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:17,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:17,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=149}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=149 2024-11-12T14:19:17,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=149 2024-11-12T14:19:17,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-12T14:19:17,641 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0470 sec 2024-11-12T14:19:17,643 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=148, table=TestAcidGuarantees in 1.0520 sec 2024-11-12T14:19:17,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=148 2024-11-12T14:19:17,729 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:19:17,730 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:19:17,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-12T14:19:17,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T14:19:17,732 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:19:17,732 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:19:17,733 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:19:17,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T14:19:17,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-12T14:19:17,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:17,885 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing e1aa16d140ed3f62a95c849e7bf6cc7a 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T14:19:17,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=A 2024-11-12T14:19:17,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:17,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=B 2024-11-12T14:19:17,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:17,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=C 2024-11-12T14:19:17,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:17,891 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112b943b5a6c4504e588069d107ad2d7acd_e1aa16d140ed3f62a95c849e7bf6cc7a is 50, key is test_row_0/A:col10/1731421156819/Put/seqid=0 2024-11-12T14:19:17,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742424_1600 (size=12304) 2024-11-12T14:19:17,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:17,961 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:17,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:17,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421217979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:17,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:17,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:17,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:17,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421217980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:17,985 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:17,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:17,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:17,986 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:17,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:17,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421217984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:17,989 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:17,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:17,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421217984, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:17,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:17,989 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:17,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:17,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:17,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:17,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:17,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421217985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:17,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:17,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:17,990 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:18,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T14:19:18,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:18,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421218090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:18,092 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:18,092 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:18,092 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:18,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:18,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421218099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:18,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:18,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421218099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:18,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:18,103 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:18,103 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:18,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:18,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:18,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421218099, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:18,103 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:18,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:18,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:18,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:18,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:18,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421218299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:18,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:18,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:18,302 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:18,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:18,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421218309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:18,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421218309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:18,313 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421218309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:18,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:19:18,321 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112b943b5a6c4504e588069d107ad2d7acd_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b943b5a6c4504e588069d107ad2d7acd_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:18,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/b7366b3a299e447289a91cc9afd3c857, store: [table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:18,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/b7366b3a299e447289a91cc9afd3c857 is 175, key is test_row_0/A:col10/1731421156819/Put/seqid=0 2024-11-12T14:19:18,325 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742425_1601 (size=31105) 2024-11-12T14:19:18,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T14:19:18,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:18,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421218609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:18,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:18,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:18,611 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:18,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:18,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421218619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:18,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:18,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421218620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:18,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:18,622 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:18,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421218620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:18,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:18,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:18,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:18,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:18,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:18,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:18,623 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:18,623 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:18,725 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/b7366b3a299e447289a91cc9afd3c857 2024-11-12T14:19:18,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/1c1f93d72d7641ef806a29895fcdeb75 is 50, key is test_row_0/B:col10/1731421156819/Put/seqid=0 2024-11-12T14:19:18,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742426_1602 (size=12151) 2024-11-12T14:19:18,743 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/1c1f93d72d7641ef806a29895fcdeb75 2024-11-12T14:19:18,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/70e52341f25a4a6ca8cb62e10fb9e994 is 50, key is test_row_0/C:col10/1731421156819/Put/seqid=0 2024-11-12T14:19:18,787 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742427_1603 (size=12151) 2024-11-12T14:19:18,787 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/70e52341f25a4a6ca8cb62e10fb9e994 2024-11-12T14:19:18,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/b7366b3a299e447289a91cc9afd3c857 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/b7366b3a299e447289a91cc9afd3c857 2024-11-12T14:19:18,797 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/b7366b3a299e447289a91cc9afd3c857, entries=150, sequenceid=134, filesize=30.4 K 2024-11-12T14:19:18,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/1c1f93d72d7641ef806a29895fcdeb75 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/1c1f93d72d7641ef806a29895fcdeb75 2024-11-12T14:19:18,803 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/1c1f93d72d7641ef806a29895fcdeb75, entries=150, sequenceid=134, filesize=11.9 K 2024-11-12T14:19:18,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/70e52341f25a4a6ca8cb62e10fb9e994 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/70e52341f25a4a6ca8cb62e10fb9e994 2024-11-12T14:19:18,808 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/70e52341f25a4a6ca8cb62e10fb9e994, entries=150, sequenceid=134, filesize=11.9 K 2024-11-12T14:19:18,809 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for e1aa16d140ed3f62a95c849e7bf6cc7a in 924ms, sequenceid=134, compaction requested=true 2024-11-12T14:19:18,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:18,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:18,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-12T14:19:18,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-12T14:19:18,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-12T14:19:18,814 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0810 sec 2024-11-12T14:19:18,816 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.0850 sec 2024-11-12T14:19:18,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-12T14:19:18,870 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:19:18,870 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:19:18,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-12T14:19:18,871 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:19:18,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T14:19:18,875 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:19:18,876 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:19:18,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T14:19:19,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:19,005 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1aa16d140ed3f62a95c849e7bf6cc7a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T14:19:19,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=A 2024-11-12T14:19:19,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:19,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=B 2024-11-12T14:19:19,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:19,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=C 2024-11-12T14:19:19,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:19,019 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112b1484f2ad20e49e9b3796f0fee2839f5_e1aa16d140ed3f62a95c849e7bf6cc7a is 50, key is test_row_0/A:col10/1731421157982/Put/seqid=0 2024-11-12T14:19:19,029 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T14:19:19,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:19,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:19,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:19,029 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:19,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:19,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:19,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742428_1604 (size=14794) 2024-11-12T14:19:19,041 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:19:19,044 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112b1484f2ad20e49e9b3796f0fee2839f5_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b1484f2ad20e49e9b3796f0fee2839f5_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:19,045 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/23c3e751e6af42b3bce1c56724d58f51, store: [table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:19,046 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/23c3e751e6af42b3bce1c56724d58f51 is 175, key is test_row_0/A:col10/1731421157982/Put/seqid=0 2024-11-12T14:19:19,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742429_1605 (size=39749) 2024-11-12T14:19:19,058 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=152, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/23c3e751e6af42b3bce1c56724d58f51 2024-11-12T14:19:19,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/8c68977682074f54adc6a38e7425b24f is 50, key is test_row_0/B:col10/1731421157982/Put/seqid=0 2024-11-12T14:19:19,072 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:19,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421219067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:19,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:19,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:19,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:19,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742430_1606 (size=12151) 2024-11-12T14:19:19,125 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:19,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421219120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:19,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:19,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:19,125 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:19,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:19,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421219129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:19,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421219129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:19,131 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:19,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421219129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:19,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:19,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:19,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:19,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:19,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:19,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:19,131 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:19,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:19,131 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:19,182 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T14:19:19,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:19,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:19,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:19,182 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:19,182 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:19,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421219180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:19,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:19,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:19,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:19,184 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:19,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T14:19:19,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T14:19:19,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:19,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:19,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:19,335 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:19,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:19,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:19,393 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:19,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421219389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:19,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:19,474 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/8c68977682074f54adc6a38e7425b24f 2024-11-12T14:19:19,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/c8f609d8605e43f6b11f6ba4844d7586 is 50, key is test_row_0/C:col10/1731421157982/Put/seqid=0 2024-11-12T14:19:19,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T14:19:19,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:19,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:19,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:19,489 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:19,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:19,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:19,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742431_1607 (size=12151) 2024-11-12T14:19:19,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T14:19:19,642 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T14:19:19,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:19,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:19,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:19,642 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:19,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:19,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:19,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:19,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421219699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:19,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:19,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:19,702 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:19,794 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T14:19:19,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:19,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:19,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:19,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:19,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:19,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:19,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=152 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/c8f609d8605e43f6b11f6ba4844d7586 2024-11-12T14:19:19,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/23c3e751e6af42b3bce1c56724d58f51 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/23c3e751e6af42b3bce1c56724d58f51 2024-11-12T14:19:19,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/23c3e751e6af42b3bce1c56724d58f51, entries=200, sequenceid=152, filesize=38.8 K 2024-11-12T14:19:19,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/8c68977682074f54adc6a38e7425b24f as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/8c68977682074f54adc6a38e7425b24f 2024-11-12T14:19:19,902 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/8c68977682074f54adc6a38e7425b24f, entries=150, sequenceid=152, filesize=11.9 K 2024-11-12T14:19:19,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/c8f609d8605e43f6b11f6ba4844d7586 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/c8f609d8605e43f6b11f6ba4844d7586 2024-11-12T14:19:19,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/c8f609d8605e43f6b11f6ba4844d7586, entries=150, sequenceid=152, filesize=11.9 K 2024-11-12T14:19:19,907 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e1aa16d140ed3f62a95c849e7bf6cc7a in 902ms, sequenceid=152, compaction requested=true 2024-11-12T14:19:19,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:19,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1aa16d140ed3f62a95c849e7bf6cc7a:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:19:19,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:19,907 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:19:19,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1aa16d140ed3f62a95c849e7bf6cc7a:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:19:19,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:19,907 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:19:19,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1aa16d140ed3f62a95c849e7bf6cc7a:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:19:19,907 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:19:19,908 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 132970 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:19:19,908 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): e1aa16d140ed3f62a95c849e7bf6cc7a/A is initiating minor compaction (all files) 2024-11-12T14:19:19,908 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1aa16d140ed3f62a95c849e7bf6cc7a/A in TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:19,908 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/f5949ebfcbee4e5687266bf370b5239e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/cba804fdea45429ab0b8658d33065559, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/b7366b3a299e447289a91cc9afd3c857, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/23c3e751e6af42b3bce1c56724d58f51] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp, totalSize=129.9 K 2024-11-12T14:19:19,908 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:19,908 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. files: [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/f5949ebfcbee4e5687266bf370b5239e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/cba804fdea45429ab0b8658d33065559, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/b7366b3a299e447289a91cc9afd3c857, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/23c3e751e6af42b3bce1c56724d58f51] 2024-11-12T14:19:19,908 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:19:19,908 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): e1aa16d140ed3f62a95c849e7bf6cc7a/B is initiating minor compaction (all files) 2024-11-12T14:19:19,908 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1aa16d140ed3f62a95c849e7bf6cc7a/B in TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:19,909 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/d59297916ed24b6ea3f63c2f014a9455, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/09385d7a5b20481cb75c65e4271772d5, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/1c1f93d72d7641ef806a29895fcdeb75, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/8c68977682074f54adc6a38e7425b24f] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp, totalSize=47.4 K 2024-11-12T14:19:19,909 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5949ebfcbee4e5687266bf370b5239e, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731421155014 2024-11-12T14:19:19,909 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting d59297916ed24b6ea3f63c2f014a9455, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731421155014 2024-11-12T14:19:19,909 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting cba804fdea45429ab0b8658d33065559, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731421156177 2024-11-12T14:19:19,909 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 09385d7a5b20481cb75c65e4271772d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731421156177 2024-11-12T14:19:19,909 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7366b3a299e447289a91cc9afd3c857, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731421156819 2024-11-12T14:19:19,909 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c1f93d72d7641ef806a29895fcdeb75, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731421156819 2024-11-12T14:19:19,909 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23c3e751e6af42b3bce1c56724d58f51, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731421157982 2024-11-12T14:19:19,909 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c68977682074f54adc6a38e7425b24f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731421157982 2024-11-12T14:19:19,933 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:19,934 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1aa16d140ed3f62a95c849e7bf6cc7a#B#compaction#516 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:19,934 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/5ffaecee499b484d91f5b61c8fc27e02 is 50, key is test_row_0/B:col10/1731421157982/Put/seqid=0 2024-11-12T14:19:19,935 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111293f0a2706a9447d0831bc980c0d993b8_e1aa16d140ed3f62a95c849e7bf6cc7a store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:19,938 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111293f0a2706a9447d0831bc980c0d993b8_e1aa16d140ed3f62a95c849e7bf6cc7a, store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:19,938 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111293f0a2706a9447d0831bc980c0d993b8_e1aa16d140ed3f62a95c849e7bf6cc7a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:19,938 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742432_1608 (size=12493) 2024-11-12T14:19:19,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742433_1609 (size=4469) 2024-11-12T14:19:19,942 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/5ffaecee499b484d91f5b61c8fc27e02 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/5ffaecee499b484d91f5b61c8fc27e02 2024-11-12T14:19:19,943 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1aa16d140ed3f62a95c849e7bf6cc7a#A#compaction#517 average throughput is 2.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:19,943 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/552c88da6b4c47b480e1603b3268ca45 is 175, key is test_row_0/A:col10/1731421157982/Put/seqid=0 2024-11-12T14:19:19,947 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742434_1610 (size=31447) 2024-11-12T14:19:19,947 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-12T14:19:19,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:19,948 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing e1aa16d140ed3f62a95c849e7bf6cc7a 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T14:19:19,948 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1aa16d140ed3f62a95c849e7bf6cc7a/B of e1aa16d140ed3f62a95c849e7bf6cc7a into 5ffaecee499b484d91f5b61c8fc27e02(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:19:19,948 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:19,948 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., storeName=e1aa16d140ed3f62a95c849e7bf6cc7a/B, priority=12, startTime=1731421159907; duration=0sec 2024-11-12T14:19:19,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=A 2024-11-12T14:19:19,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:19,948 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:19:19,948 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1aa16d140ed3f62a95c849e7bf6cc7a:B 2024-11-12T14:19:19,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=B 2024-11-12T14:19:19,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:19,948 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:19:19,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=C 2024-11-12T14:19:19,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:19,951 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48510 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:19:19,951 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): e1aa16d140ed3f62a95c849e7bf6cc7a/C is initiating minor compaction (all files) 2024-11-12T14:19:19,951 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1aa16d140ed3f62a95c849e7bf6cc7a/C in TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:19,951 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/03f2be16f5f843929d8b25d276f94d88, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/7a7a04b7327f4ea880f837a30e43e33b, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/70e52341f25a4a6ca8cb62e10fb9e994, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/c8f609d8605e43f6b11f6ba4844d7586] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp, totalSize=47.4 K 2024-11-12T14:19:19,951 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 03f2be16f5f843929d8b25d276f94d88, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=98, earliestPutTs=1731421155014 2024-11-12T14:19:19,952 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a7a04b7327f4ea880f837a30e43e33b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731421156177 2024-11-12T14:19:19,952 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 70e52341f25a4a6ca8cb62e10fb9e994, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731421156819 2024-11-12T14:19:19,952 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting c8f609d8605e43f6b11f6ba4844d7586, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731421157982 2024-11-12T14:19:19,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e701ae80a27646cba31229d48e19bea3_e1aa16d140ed3f62a95c849e7bf6cc7a is 50, key is test_row_0/A:col10/1731421159059/Put/seqid=0 2024-11-12T14:19:19,960 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1aa16d140ed3f62a95c849e7bf6cc7a#C#compaction#519 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:19,961 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/2f3da4e666bd4680a2b4266df8ceb003 is 50, key is test_row_0/C:col10/1731421157982/Put/seqid=0 2024-11-12T14:19:19,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742435_1611 (size=12304) 2024-11-12T14:19:19,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:19:19,966 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e701ae80a27646cba31229d48e19bea3_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e701ae80a27646cba31229d48e19bea3_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:19,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/f6657545e8d64d4eb2d61e846329fbc4, store: [table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:19,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/f6657545e8d64d4eb2d61e846329fbc4 is 175, key is test_row_0/A:col10/1731421159059/Put/seqid=0 2024-11-12T14:19:19,974 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742437_1613 (size=31105) 2024-11-12T14:19:19,975 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=167, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/f6657545e8d64d4eb2d61e846329fbc4 2024-11-12T14:19:19,987 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742436_1612 (size=12493) 2024-11-12T14:19:19,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/cc6210e6e97943af9ced36691c6f925d is 50, key is test_row_0/B:col10/1731421159059/Put/seqid=0 2024-11-12T14:19:19,997 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/2f3da4e666bd4680a2b4266df8ceb003 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/2f3da4e666bd4680a2b4266df8ceb003 2024-11-12T14:19:20,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742438_1614 (size=12151) 2024-11-12T14:19:20,002 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1aa16d140ed3f62a95c849e7bf6cc7a/C of e1aa16d140ed3f62a95c849e7bf6cc7a into 2f3da4e666bd4680a2b4266df8ceb003(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:19:20,002 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:20,002 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., storeName=e1aa16d140ed3f62a95c849e7bf6cc7a/C, priority=12, startTime=1731421159907; duration=0sec 2024-11-12T14:19:20,002 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:20,002 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1aa16d140ed3f62a95c849e7bf6cc7a:C 2024-11-12T14:19:20,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T14:19:20,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:20,137 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:20,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:20,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421220162, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:20,165 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:20,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421220163, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:20,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,165 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:20,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,166 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:20,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:20,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421220164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:20,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:20,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:20,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421220165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:20,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:20,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:20,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421220209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:20,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:20,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:20,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421220270, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:20,275 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,275 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,275 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:20,275 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:20,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421220271, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:20,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:20,281 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:20,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421220279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:20,282 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:20,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421220279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:20,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,282 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:20,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,282 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:20,335 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-12T14:19:20,335 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-12T14:19:20,362 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/552c88da6b4c47b480e1603b3268ca45 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/552c88da6b4c47b480e1603b3268ca45 2024-11-12T14:19:20,374 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1aa16d140ed3f62a95c849e7bf6cc7a/A of e1aa16d140ed3f62a95c849e7bf6cc7a into 552c88da6b4c47b480e1603b3268ca45(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:19:20,374 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:20,374 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., storeName=e1aa16d140ed3f62a95c849e7bf6cc7a/A, priority=12, startTime=1731421159907; duration=0sec 2024-11-12T14:19:20,374 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:20,374 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1aa16d140ed3f62a95c849e7bf6cc7a:A 2024-11-12T14:19:20,404 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/cc6210e6e97943af9ced36691c6f925d 2024-11-12T14:19:20,426 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/c6a2bb18b5584c0c8c4051d57bb3bb92 is 50, key is test_row_0/C:col10/1731421159059/Put/seqid=0 2024-11-12T14:19:20,456 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742439_1615 (size=12151) 2024-11-12T14:19:20,457 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/c6a2bb18b5584c0c8c4051d57bb3bb92 2024-11-12T14:19:20,465 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/f6657545e8d64d4eb2d61e846329fbc4 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/f6657545e8d64d4eb2d61e846329fbc4 2024-11-12T14:19:20,472 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/f6657545e8d64d4eb2d61e846329fbc4, entries=150, sequenceid=167, filesize=30.4 K 2024-11-12T14:19:20,475 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/cc6210e6e97943af9ced36691c6f925d as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/cc6210e6e97943af9ced36691c6f925d 2024-11-12T14:19:20,480 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/cc6210e6e97943af9ced36691c6f925d, entries=150, sequenceid=167, filesize=11.9 K 2024-11-12T14:19:20,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421220480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:20,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:20,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421220480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:20,483 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,484 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:20,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,484 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:20,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/c6a2bb18b5584c0c8c4051d57bb3bb92 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/c6a2bb18b5584c0c8c4051d57bb3bb92 2024-11-12T14:19:20,490 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/c6a2bb18b5584c0c8c4051d57bb3bb92, entries=150, sequenceid=167, filesize=11.9 K 2024-11-12T14:19:20,492 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e1aa16d140ed3f62a95c849e7bf6cc7a in 544ms, sequenceid=167, compaction requested=false 2024-11-12T14:19:20,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:20,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:20,493 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-12T14:19:20,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-12T14:19:20,496 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-12T14:19:20,497 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6190 sec 2024-11-12T14:19:20,499 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.6270 sec 2024-11-12T14:19:20,501 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1aa16d140ed3f62a95c849e7bf6cc7a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T14:19:20,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=A 2024-11-12T14:19:20,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:20,501 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=B 2024-11-12T14:19:20,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:20,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=C 2024-11-12T14:19:20,502 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:20,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:20,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111298bfcd08b1e0465289c02fddf5b8fd92_e1aa16d140ed3f62a95c849e7bf6cc7a is 50, key is test_row_0/A:col10/1731421160148/Put/seqid=0 2024-11-12T14:19:20,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:20,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421220536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:20,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,541 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:20,542 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:20,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421220539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:20,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,542 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:20,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742440_1616 (size=17284) 2024-11-12T14:19:20,587 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:19:20,591 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111298bfcd08b1e0465289c02fddf5b8fd92_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111298bfcd08b1e0465289c02fddf5b8fd92_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:20,592 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/a5446c66869d4f9588d19f2cf748b24f, store: [table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:20,592 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/a5446c66869d4f9588d19f2cf748b24f is 175, key is test_row_0/A:col10/1731421160148/Put/seqid=0 2024-11-12T14:19:20,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742441_1617 (size=48389) 2024-11-12T14:19:20,627 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=189, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/a5446c66869d4f9588d19f2cf748b24f 2024-11-12T14:19:20,646 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/d0ceee2501ba4e518b68b55238db04fd is 50, key is test_row_0/B:col10/1731421160148/Put/seqid=0 2024-11-12T14:19:20,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:20,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421220649, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:20,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,653 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:20,653 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:20,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421220650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:20,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,654 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:20,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742442_1618 (size=12151) 2024-11-12T14:19:20,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/d0ceee2501ba4e518b68b55238db04fd 2024-11-12T14:19:20,702 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/8373c2c6b3514c2f899dd23aab7c3937 is 50, key is test_row_0/C:col10/1731421160148/Put/seqid=0 2024-11-12T14:19:20,718 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742443_1619 (size=12151) 2024-11-12T14:19:20,718 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=189 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/8373c2c6b3514c2f899dd23aab7c3937 2024-11-12T14:19:20,725 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/a5446c66869d4f9588d19f2cf748b24f as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/a5446c66869d4f9588d19f2cf748b24f 2024-11-12T14:19:20,730 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/a5446c66869d4f9588d19f2cf748b24f, entries=250, sequenceid=189, filesize=47.3 K 2024-11-12T14:19:20,730 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/d0ceee2501ba4e518b68b55238db04fd as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/d0ceee2501ba4e518b68b55238db04fd 2024-11-12T14:19:20,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/d0ceee2501ba4e518b68b55238db04fd, entries=150, sequenceid=189, filesize=11.9 K 2024-11-12T14:19:20,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/8373c2c6b3514c2f899dd23aab7c3937 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/8373c2c6b3514c2f899dd23aab7c3937 2024-11-12T14:19:20,744 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/8373c2c6b3514c2f899dd23aab7c3937, entries=150, sequenceid=189, filesize=11.9 K 2024-11-12T14:19:20,744 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for e1aa16d140ed3f62a95c849e7bf6cc7a in 243ms, sequenceid=189, compaction requested=true 2024-11-12T14:19:20,744 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:20,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1aa16d140ed3f62a95c849e7bf6cc7a:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:19:20,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:20,745 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:19:20,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1aa16d140ed3f62a95c849e7bf6cc7a:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:19:20,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:19:20,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1aa16d140ed3f62a95c849e7bf6cc7a:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:19:20,745 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-12T14:19:20,745 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:19:20,746 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110941 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:19:20,746 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:19:20,746 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): e1aa16d140ed3f62a95c849e7bf6cc7a/A is initiating minor compaction (all files) 2024-11-12T14:19:20,746 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): e1aa16d140ed3f62a95c849e7bf6cc7a/B is initiating minor compaction (all files) 2024-11-12T14:19:20,746 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1aa16d140ed3f62a95c849e7bf6cc7a/A in TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:20,746 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1aa16d140ed3f62a95c849e7bf6cc7a/B in TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:20,746 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/552c88da6b4c47b480e1603b3268ca45, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/f6657545e8d64d4eb2d61e846329fbc4, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/a5446c66869d4f9588d19f2cf748b24f] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp, totalSize=108.3 K 2024-11-12T14:19:20,746 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/5ffaecee499b484d91f5b61c8fc27e02, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/cc6210e6e97943af9ced36691c6f925d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/d0ceee2501ba4e518b68b55238db04fd] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp, totalSize=35.9 K 2024-11-12T14:19:20,746 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:20,746 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. files: [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/552c88da6b4c47b480e1603b3268ca45, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/f6657545e8d64d4eb2d61e846329fbc4, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/a5446c66869d4f9588d19f2cf748b24f] 2024-11-12T14:19:20,747 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 552c88da6b4c47b480e1603b3268ca45, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731421157982 2024-11-12T14:19:20,747 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 5ffaecee499b484d91f5b61c8fc27e02, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731421157982 2024-11-12T14:19:20,747 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting cc6210e6e97943af9ced36691c6f925d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731421159037 2024-11-12T14:19:20,747 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting f6657545e8d64d4eb2d61e846329fbc4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731421159037 2024-11-12T14:19:20,747 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting d0ceee2501ba4e518b68b55238db04fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731421160148 2024-11-12T14:19:20,749 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5446c66869d4f9588d19f2cf748b24f, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731421160148 2024-11-12T14:19:20,763 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1aa16d140ed3f62a95c849e7bf6cc7a#B#compaction#525 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:20,764 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/7c8c639943b14904ae5741a87f187db5 is 50, key is test_row_0/B:col10/1731421160148/Put/seqid=0 2024-11-12T14:19:20,765 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:20,785 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241112e8c2de6101e34575984b8b5b2d59f3bc_e1aa16d140ed3f62a95c849e7bf6cc7a store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:20,787 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241112e8c2de6101e34575984b8b5b2d59f3bc_e1aa16d140ed3f62a95c849e7bf6cc7a, store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:20,787 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e8c2de6101e34575984b8b5b2d59f3bc_e1aa16d140ed3f62a95c849e7bf6cc7a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:20,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:20,793 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1aa16d140ed3f62a95c849e7bf6cc7a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T14:19:20,793 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=A 2024-11-12T14:19:20,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:20,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=B 2024-11-12T14:19:20,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:20,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=C 2024-11-12T14:19:20,794 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:20,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742444_1620 (size=12595) 2024-11-12T14:19:20,808 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/7c8c639943b14904ae5741a87f187db5 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/7c8c639943b14904ae5741a87f187db5 2024-11-12T14:19:20,814 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1aa16d140ed3f62a95c849e7bf6cc7a/B of e1aa16d140ed3f62a95c849e7bf6cc7a into 7c8c639943b14904ae5741a87f187db5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:19:20,814 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:20,814 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., storeName=e1aa16d140ed3f62a95c849e7bf6cc7a/B, priority=13, startTime=1731421160745; duration=0sec 2024-11-12T14:19:20,814 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:19:20,814 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1aa16d140ed3f62a95c849e7bf6cc7a:B 2024-11-12T14:19:20,814 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:19:20,816 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:19:20,816 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): e1aa16d140ed3f62a95c849e7bf6cc7a/C is initiating minor compaction (all files) 2024-11-12T14:19:20,817 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1aa16d140ed3f62a95c849e7bf6cc7a/C in TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:20,817 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/2f3da4e666bd4680a2b4266df8ceb003, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/c6a2bb18b5584c0c8c4051d57bb3bb92, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/8373c2c6b3514c2f899dd23aab7c3937] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp, totalSize=35.9 K 2024-11-12T14:19:20,817 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 2f3da4e666bd4680a2b4266df8ceb003, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=152, earliestPutTs=1731421157982 2024-11-12T14:19:20,828 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting c6a2bb18b5584c0c8c4051d57bb3bb92, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1731421159037 2024-11-12T14:19:20,829 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 8373c2c6b3514c2f899dd23aab7c3937, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731421160148 2024-11-12T14:19:20,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742445_1621 (size=4469) 2024-11-12T14:19:20,830 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1aa16d140ed3f62a95c849e7bf6cc7a#A#compaction#526 average throughput is 0.38 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:20,831 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/b007d9ed20ad4a169b68488e2b7a8452 is 175, key is test_row_0/A:col10/1731421160148/Put/seqid=0 2024-11-12T14:19:20,838 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411124cd01ccc6f554cdb8d4d29afcb5a2bc4_e1aa16d140ed3f62a95c849e7bf6cc7a is 50, key is test_row_0/A:col10/1731421160792/Put/seqid=0 2024-11-12T14:19:20,840 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:20,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421220836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:20,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,841 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:20,842 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:20,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421220839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:20,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,843 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:20,846 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1aa16d140ed3f62a95c849e7bf6cc7a#C#compaction#528 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:20,847 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/299802d71c914b879d06b2e97b8bfbc1 is 50, key is test_row_0/C:col10/1731421160148/Put/seqid=0 2024-11-12T14:19:20,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:20,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421220859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:20,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:20,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421220859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:20,863 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:20,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742447_1623 (size=14794) 2024-11-12T14:19:20,893 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742446_1622 (size=31549) 2024-11-12T14:19:20,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742448_1624 (size=12595) 2024-11-12T14:19:20,900 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/b007d9ed20ad4a169b68488e2b7a8452 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/b007d9ed20ad4a169b68488e2b7a8452 2024-11-12T14:19:20,904 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1aa16d140ed3f62a95c849e7bf6cc7a/A of e1aa16d140ed3f62a95c849e7bf6cc7a into b007d9ed20ad4a169b68488e2b7a8452(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:19:20,904 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:20,904 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., storeName=e1aa16d140ed3f62a95c849e7bf6cc7a/A, priority=13, startTime=1731421160745; duration=0sec 2024-11-12T14:19:20,904 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:20,904 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1aa16d140ed3f62a95c849e7bf6cc7a:A 2024-11-12T14:19:20,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:20,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421220949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:20,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:20,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,952 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:20,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421220949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:20,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:20,952 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:21,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-12T14:19:21,019 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:19:21,020 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:19:21,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-12T14:19:21,021 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:19:21,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T14:19:21,022 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:19:21,022 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:19:21,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T14:19:21,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:21,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421221160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:21,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:21,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:21,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421221160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:21,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:21,164 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:21,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:21,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:21,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:21,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:21,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421221169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:21,172 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:21,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421221169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:21,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:21,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:21,173 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:21,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:21,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:21,173 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:21,178 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T14:19:21,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:21,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:21,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:21,178 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:21,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:21,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:21,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:21,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421221229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:21,230 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_0' in e1aa16d140ed3f62a95c849e7bf6cc7a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:19:21,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:21,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:21,231 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:21,284 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:19:21,287 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411124cd01ccc6f554cdb8d4d29afcb5a2bc4_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124cd01ccc6f554cdb8d4d29afcb5a2bc4_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:21,287 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/2d6bef64705644ac842b5c9ad57af5ca, store: [table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:21,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/2d6bef64705644ac842b5c9ad57af5ca is 175, key is test_row_0/A:col10/1731421160792/Put/seqid=0 2024-11-12T14:19:21,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742449_1625 (size=39749) 2024-11-12T14:19:21,301 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=203, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/2d6bef64705644ac842b5c9ad57af5ca 2024-11-12T14:19:21,302 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/299802d71c914b879d06b2e97b8bfbc1 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/299802d71c914b879d06b2e97b8bfbc1 2024-11-12T14:19:21,307 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1aa16d140ed3f62a95c849e7bf6cc7a/C of e1aa16d140ed3f62a95c849e7bf6cc7a into 299802d71c914b879d06b2e97b8bfbc1(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:19:21,307 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:21,307 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., storeName=e1aa16d140ed3f62a95c849e7bf6cc7a/C, priority=13, startTime=1731421160745; duration=0sec 2024-11-12T14:19:21,307 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:21,307 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1aa16d140ed3f62a95c849e7bf6cc7a:C 2024-11-12T14:19:21,308 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/bf9635e8dbd44c77afccb69e08035c3e is 50, key is test_row_0/B:col10/1731421160792/Put/seqid=0 2024-11-12T14:19:21,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742450_1626 (size=12151) 2024-11-12T14:19:21,331 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T14:19:21,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:21,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:21,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:21,331 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:21,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:21,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:21,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T14:19:21,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:21,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421221469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:21,470 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:21,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:21,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:21,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421221469, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:21,470 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:21,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:21,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:21,471 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:21,484 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T14:19:21,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:21,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:21,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:21,485 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:21,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:21,639 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T14:19:21,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:21,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:21,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:21,640 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:21,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:21,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:21,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T14:19:21,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:21,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421221683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:21,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:21,684 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:21,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:21,684 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:21,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421221683, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:21,684 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:21,685 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:21,717 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/bf9635e8dbd44c77afccb69e08035c3e 2024-11-12T14:19:21,726 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/c4bfd34593634b909c19fdf35ca9c562 is 50, key is test_row_0/C:col10/1731421160792/Put/seqid=0 2024-11-12T14:19:21,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742451_1627 (size=12151) 2024-11-12T14:19:21,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T14:19:21,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:21,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:21,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:21,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:21,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:21,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:21,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T14:19:21,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:21,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:21,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:21,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:21,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:21,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:21,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421221979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:21,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:21,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:21,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:21,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421221980, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:21,982 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:21,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:21,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:21,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:22,098 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T14:19:22,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:22,098 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:22,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:22,099 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:22,099 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:22,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:22,155 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/c4bfd34593634b909c19fdf35ca9c562 2024-11-12T14:19:22,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/2d6bef64705644ac842b5c9ad57af5ca as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/2d6bef64705644ac842b5c9ad57af5ca 2024-11-12T14:19:22,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T14:19:22,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/2d6bef64705644ac842b5c9ad57af5ca, entries=200, sequenceid=203, filesize=38.8 K 2024-11-12T14:19:22,162 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/bf9635e8dbd44c77afccb69e08035c3e as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/bf9635e8dbd44c77afccb69e08035c3e 2024-11-12T14:19:22,165 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/bf9635e8dbd44c77afccb69e08035c3e, entries=150, sequenceid=203, filesize=11.9 K 2024-11-12T14:19:22,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/c4bfd34593634b909c19fdf35ca9c562 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/c4bfd34593634b909c19fdf35ca9c562 2024-11-12T14:19:22,168 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/c4bfd34593634b909c19fdf35ca9c562, entries=150, sequenceid=203, filesize=11.9 K 2024-11-12T14:19:22,169 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e1aa16d140ed3f62a95c849e7bf6cc7a in 1376ms, sequenceid=203, compaction requested=false 2024-11-12T14:19:22,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:22,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-12T14:19:22,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:22,251 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing e1aa16d140ed3f62a95c849e7bf6cc7a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T14:19:22,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=A 2024-11-12T14:19:22,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:22,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=B 2024-11-12T14:19:22,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:22,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=C 2024-11-12T14:19:22,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:22,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111241ba9ebab9604b2faea227409f2fe620_e1aa16d140ed3f62a95c849e7bf6cc7a is 50, key is test_row_0/A:col10/1731421160833/Put/seqid=0 2024-11-12T14:19:22,265 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742452_1628 (size=12304) 2024-11-12T14:19:22,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:19:22,270 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111241ba9ebab9604b2faea227409f2fe620_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111241ba9ebab9604b2faea227409f2fe620_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:22,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/4ce938e760a04a3baca9b480ea5f52f0, store: [table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:22,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/4ce938e760a04a3baca9b480ea5f52f0 is 175, key is test_row_0/A:col10/1731421160833/Put/seqid=0 2024-11-12T14:19:22,288 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742453_1629 (size=31105) 2024-11-12T14:19:22,686 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=224, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/4ce938e760a04a3baca9b480ea5f52f0 2024-11-12T14:19:22,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/92ed1228396247bf894d2c3929ce14e3 is 50, key is test_row_0/B:col10/1731421160833/Put/seqid=0 2024-11-12T14:19:22,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:22,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742454_1630 (size=12151) 2024-11-12T14:19:22,698 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/92ed1228396247bf894d2c3929ce14e3 2024-11-12T14:19:22,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:22,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/2efbf91ce4e34786b7244800674cf969 is 50, key is test_row_0/C:col10/1731421160833/Put/seqid=0 2024-11-12T14:19:22,735 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:22,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421222735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:22,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:22,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:22,736 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:22,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742455_1631 (size=12151) 2024-11-12T14:19:22,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:22,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421222737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:22,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:22,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:22,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:22,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:22,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421222839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:22,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:22,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:22,842 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:22,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:22,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421222849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:22,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:22,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:22,853 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:22,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:22,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421222989, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:22,994 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_1' in e1aa16d140ed3f62a95c849e7bf6cc7a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:19:22,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:22,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:22,995 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:23,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:23,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421223000, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:23,005 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_0' in e1aa16d140ed3f62a95c849e7bf6cc7a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:19:23,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:23,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:23,006 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:23,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:23,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421223049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:23,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:23,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:23,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:23,060 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:23,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421223059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:23,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:23,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:23,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:23,139 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=224 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/2efbf91ce4e34786b7244800674cf969 2024-11-12T14:19:23,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/4ce938e760a04a3baca9b480ea5f52f0 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/4ce938e760a04a3baca9b480ea5f52f0 2024-11-12T14:19:23,158 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/4ce938e760a04a3baca9b480ea5f52f0, entries=150, sequenceid=224, filesize=30.4 K 2024-11-12T14:19:23,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/92ed1228396247bf894d2c3929ce14e3 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/92ed1228396247bf894d2c3929ce14e3 2024-11-12T14:19:23,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T14:19:23,177 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/92ed1228396247bf894d2c3929ce14e3, entries=150, sequenceid=224, filesize=11.9 K 2024-11-12T14:19:23,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/2efbf91ce4e34786b7244800674cf969 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/2efbf91ce4e34786b7244800674cf969 2024-11-12T14:19:23,199 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/2efbf91ce4e34786b7244800674cf969, entries=150, sequenceid=224, filesize=11.9 K 2024-11-12T14:19:23,200 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e1aa16d140ed3f62a95c849e7bf6cc7a in 949ms, sequenceid=224, compaction requested=true 2024-11-12T14:19:23,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:23,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:23,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-12T14:19:23,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-12T14:19:23,204 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-12T14:19:23,204 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1800 sec 2024-11-12T14:19:23,208 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.1850 sec 2024-11-12T14:19:23,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:23,247 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1aa16d140ed3f62a95c849e7bf6cc7a 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T14:19:23,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=A 2024-11-12T14:19:23,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:23,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=B 2024-11-12T14:19:23,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:23,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=C 2024-11-12T14:19:23,247 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:23,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125cfa2851b5784cc995c3dee400a32604_e1aa16d140ed3f62a95c849e7bf6cc7a is 50, key is test_row_0/A:col10/1731421163245/Put/seqid=0 2024-11-12T14:19:23,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742456_1632 (size=14794) 2024-11-12T14:19:23,365 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:19:23,387 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:23,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421223380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:23,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:23,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:23,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:23,388 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:23,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421223381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:23,388 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:23,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:23,389 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:23,395 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:23,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421223384, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:23,395 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:23,406 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125cfa2851b5784cc995c3dee400a32604_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125cfa2851b5784cc995c3dee400a32604_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:23,408 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/69b761b4ab84467387b52f7b2a75f56b, store: [table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:23,409 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/69b761b4ab84467387b52f7b2a75f56b is 175, key is test_row_0/A:col10/1731421163245/Put/seqid=0 2024-11-12T14:19:23,457 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742457_1633 (size=39749) 2024-11-12T14:19:23,472 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/69b761b4ab84467387b52f7b2a75f56b 2024-11-12T14:19:23,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/7560f58b05ac4ab599f3939240c84f90 is 50, key is test_row_0/B:col10/1731421163245/Put/seqid=0 2024-11-12T14:19:23,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421223501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:23,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:23,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421223501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:23,508 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:23,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421223501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:23,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:23,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:23,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:23,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:23,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:23,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:23,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:23,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:23,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:23,538 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742458_1634 (size=12151) 2024-11-12T14:19:23,540 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/7560f58b05ac4ab599f3939240c84f90 2024-11-12T14:19:23,551 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/3c892856a58241a98d054b6ae0857b87 is 50, key is test_row_0/C:col10/1731421163245/Put/seqid=0 2024-11-12T14:19:23,580 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742459_1635 (size=12151) 2024-11-12T14:19:23,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:23,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421223710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:23,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:23,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:23,711 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:23,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421223721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:23,724 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:23,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421223722, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:23,724 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:23,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:23,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:23,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:23,996 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/3c892856a58241a98d054b6ae0857b87 2024-11-12T14:19:24,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/69b761b4ab84467387b52f7b2a75f56b as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/69b761b4ab84467387b52f7b2a75f56b 2024-11-12T14:19:24,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:24,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421224020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:24,023 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/69b761b4ab84467387b52f7b2a75f56b, entries=200, sequenceid=239, filesize=38.8 K 2024-11-12T14:19:24,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:24,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:24,024 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:24,024 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/7560f58b05ac4ab599f3939240c84f90 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/7560f58b05ac4ab599f3939240c84f90 2024-11-12T14:19:24,027 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/7560f58b05ac4ab599f3939240c84f90, entries=150, sequenceid=239, filesize=11.9 K 2024-11-12T14:19:24,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/3c892856a58241a98d054b6ae0857b87 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/3c892856a58241a98d054b6ae0857b87 2024-11-12T14:19:24,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/3c892856a58241a98d054b6ae0857b87, entries=150, sequenceid=239, filesize=11.9 K 2024-11-12T14:19:24,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~239.77 KB/245520, currentSize=93.93 KB/96180 for e1aa16d140ed3f62a95c849e7bf6cc7a in 794ms, sequenceid=239, compaction requested=true 2024-11-12T14:19:24,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:24,041 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:19:24,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1aa16d140ed3f62a95c849e7bf6cc7a:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:19:24,041 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:24,041 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:19:24,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1aa16d140ed3f62a95c849e7bf6cc7a:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:19:24,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:24,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1aa16d140ed3f62a95c849e7bf6cc7a:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:19:24,042 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:19:24,045 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142152 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:19:24,045 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:19:24,045 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): e1aa16d140ed3f62a95c849e7bf6cc7a/A is initiating minor compaction (all files) 2024-11-12T14:19:24,045 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): e1aa16d140ed3f62a95c849e7bf6cc7a/B is initiating minor compaction (all files) 2024-11-12T14:19:24,045 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1aa16d140ed3f62a95c849e7bf6cc7a/B in TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:24,045 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1aa16d140ed3f62a95c849e7bf6cc7a/A in TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:24,046 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/b007d9ed20ad4a169b68488e2b7a8452, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/2d6bef64705644ac842b5c9ad57af5ca, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/4ce938e760a04a3baca9b480ea5f52f0, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/69b761b4ab84467387b52f7b2a75f56b] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp, totalSize=138.8 K 2024-11-12T14:19:24,046 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/7c8c639943b14904ae5741a87f187db5, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/bf9635e8dbd44c77afccb69e08035c3e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/92ed1228396247bf894d2c3929ce14e3, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/7560f58b05ac4ab599f3939240c84f90] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp, totalSize=47.9 K 2024-11-12T14:19:24,046 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:24,046 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. files: [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/b007d9ed20ad4a169b68488e2b7a8452, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/2d6bef64705644ac842b5c9ad57af5ca, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/4ce938e760a04a3baca9b480ea5f52f0, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/69b761b4ab84467387b52f7b2a75f56b] 2024-11-12T14:19:24,046 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b007d9ed20ad4a169b68488e2b7a8452, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731421160148 2024-11-12T14:19:24,046 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c8c639943b14904ae5741a87f187db5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731421160148 2024-11-12T14:19:24,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:24,046 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1aa16d140ed3f62a95c849e7bf6cc7a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T14:19:24,046 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting bf9635e8dbd44c77afccb69e08035c3e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731421160525 2024-11-12T14:19:24,046 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d6bef64705644ac842b5c9ad57af5ca, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731421160525 2024-11-12T14:19:24,047 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 92ed1228396247bf894d2c3929ce14e3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731421160830 2024-11-12T14:19:24,047 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=A 2024-11-12T14:19:24,047 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ce938e760a04a3baca9b480ea5f52f0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731421160830 2024-11-12T14:19:24,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:24,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=B 2024-11-12T14:19:24,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:24,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=C 2024-11-12T14:19:24,050 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 7560f58b05ac4ab599f3939240c84f90, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731421162721 2024-11-12T14:19:24,050 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:24,051 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 69b761b4ab84467387b52f7b2a75f56b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731421162721 2024-11-12T14:19:24,080 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112909780ea756b4440af5cdaa8f393bdde_e1aa16d140ed3f62a95c849e7bf6cc7a is 50, key is test_row_0/A:col10/1731421163378/Put/seqid=0 2024-11-12T14:19:24,123 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1aa16d140ed3f62a95c849e7bf6cc7a#B#compaction#538 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:24,124 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/96365ebadcf046978831bbf67cccd0ed is 50, key is test_row_0/B:col10/1731421163245/Put/seqid=0 2024-11-12T14:19:24,125 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:24,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:24,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421224129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:24,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:24,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:24,138 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:24,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742460_1636 (size=12354) 2024-11-12T14:19:24,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:24,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421224140, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:24,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:24,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:24,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:24,153 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411127a3a74ffc825431ba0a2ab50efaeb678_e1aa16d140ed3f62a95c849e7bf6cc7a store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:24,156 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411127a3a74ffc825431ba0a2ab50efaeb678_e1aa16d140ed3f62a95c849e7bf6cc7a, store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:24,156 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411127a3a74ffc825431ba0a2ab50efaeb678_e1aa16d140ed3f62a95c849e7bf6cc7a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:24,204 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742461_1637 (size=12731) 2024-11-12T14:19:24,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742462_1638 (size=4469) 2024-11-12T14:19:24,254 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:24,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421224249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:24,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:24,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:24,254 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:24,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:24,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421224264, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:24,272 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:24,463 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:24,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421224460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:24,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:24,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:24,464 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:24,487 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:24,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421224480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:24,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:24,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:24,488 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:24,533 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:24,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421224530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:24,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:24,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:24,533 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:24,546 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:19:24,563 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112909780ea756b4440af5cdaa8f393bdde_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112909780ea756b4440af5cdaa8f393bdde_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:24,564 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/e40542cdd9aa45e8b5b52fb789ff630c, store: [table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:24,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/e40542cdd9aa45e8b5b52fb789ff630c is 175, key is test_row_0/A:col10/1731421163378/Put/seqid=0 2024-11-12T14:19:24,625 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1aa16d140ed3f62a95c849e7bf6cc7a#A#compaction#539 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:24,626 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/454275c5db4b440eae27417a788332c5 is 175, key is test_row_0/A:col10/1731421163245/Put/seqid=0 2024-11-12T14:19:24,628 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/96365ebadcf046978831bbf67cccd0ed as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/96365ebadcf046978831bbf67cccd0ed 2024-11-12T14:19:24,637 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1aa16d140ed3f62a95c849e7bf6cc7a/B of e1aa16d140ed3f62a95c849e7bf6cc7a into 96365ebadcf046978831bbf67cccd0ed(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:19:24,637 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:24,637 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., storeName=e1aa16d140ed3f62a95c849e7bf6cc7a/B, priority=12, startTime=1731421164041; duration=0sec 2024-11-12T14:19:24,638 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:19:24,638 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1aa16d140ed3f62a95c849e7bf6cc7a:B 2024-11-12T14:19:24,638 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:19:24,639 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:19:24,639 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): e1aa16d140ed3f62a95c849e7bf6cc7a/C is initiating minor compaction (all files) 2024-11-12T14:19:24,639 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1aa16d140ed3f62a95c849e7bf6cc7a/C in TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:24,639 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/299802d71c914b879d06b2e97b8bfbc1, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/c4bfd34593634b909c19fdf35ca9c562, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/2efbf91ce4e34786b7244800674cf969, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/3c892856a58241a98d054b6ae0857b87] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp, totalSize=47.9 K 2024-11-12T14:19:24,639 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 299802d71c914b879d06b2e97b8bfbc1, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=189, earliestPutTs=1731421160148 2024-11-12T14:19:24,639 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting c4bfd34593634b909c19fdf35ca9c562, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731421160525 2024-11-12T14:19:24,640 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 2efbf91ce4e34786b7244800674cf969, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=224, earliestPutTs=1731421160830 2024-11-12T14:19:24,640 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c892856a58241a98d054b6ae0857b87, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731421162721 2024-11-12T14:19:24,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742463_1639 (size=31155) 2024-11-12T14:19:24,657 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1aa16d140ed3f62a95c849e7bf6cc7a#C#compaction#540 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:24,657 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/35df06708ba6438b91692f9325cea0e6 is 50, key is test_row_0/C:col10/1731421163245/Put/seqid=0 2024-11-12T14:19:24,667 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742464_1640 (size=31685) 2024-11-12T14:19:24,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742465_1641 (size=12731) 2024-11-12T14:19:24,672 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/35df06708ba6438b91692f9325cea0e6 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/35df06708ba6438b91692f9325cea0e6 2024-11-12T14:19:24,678 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1aa16d140ed3f62a95c849e7bf6cc7a/C of e1aa16d140ed3f62a95c849e7bf6cc7a into 35df06708ba6438b91692f9325cea0e6(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:19:24,678 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:24,678 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., storeName=e1aa16d140ed3f62a95c849e7bf6cc7a/C, priority=12, startTime=1731421164042; duration=0sec 2024-11-12T14:19:24,678 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:24,678 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1aa16d140ed3f62a95c849e7bf6cc7a:C 2024-11-12T14:19:24,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:24,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421224770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:24,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:24,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:24,779 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:24,803 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:24,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421224803, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:24,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:24,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:24,804 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:25,013 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:25,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421225009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:25,014 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_1' in e1aa16d140ed3f62a95c849e7bf6cc7a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4178 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:19:25,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:25,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:25,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:25,032 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:25,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421225029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:25,033 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_0' in e1aa16d140ed3f62a95c849e7bf6cc7a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:19:25,033 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:25,033 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:25,034 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:25,041 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=257, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/e40542cdd9aa45e8b5b52fb789ff630c 2024-11-12T14:19:25,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/ab5cbd9d6da14b5c99285c65f36bd5d4 is 50, key is test_row_0/B:col10/1731421163378/Put/seqid=0 2024-11-12T14:19:25,092 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742466_1642 (size=12201) 2024-11-12T14:19:25,101 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/454275c5db4b440eae27417a788332c5 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/454275c5db4b440eae27417a788332c5 2024-11-12T14:19:25,129 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1aa16d140ed3f62a95c849e7bf6cc7a/A of e1aa16d140ed3f62a95c849e7bf6cc7a into 454275c5db4b440eae27417a788332c5(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 1sec to execute. 2024-11-12T14:19:25,129 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:25,129 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., storeName=e1aa16d140ed3f62a95c849e7bf6cc7a/A, priority=12, startTime=1731421164041; duration=1sec 2024-11-12T14:19:25,129 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:25,129 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1aa16d140ed3f62a95c849e7bf6cc7a:A 2024-11-12T14:19:25,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-12T14:19:25,179 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:19:25,180 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:19:25,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-12T14:19:25,182 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:19:25,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-12T14:19:25,183 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:19:25,184 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:19:25,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-12T14:19:25,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:25,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421225289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:25,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:25,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:25,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:25,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:25,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421225312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:25,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:25,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:25,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:25,335 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-12T14:19:25,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:25,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:25,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:25,336 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:25,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:25,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:25,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-12T14:19:25,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:25,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:25,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:25,491 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:25,491 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:25,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:25,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/ab5cbd9d6da14b5c99285c65f36bd5d4 2024-11-12T14:19:25,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-12T14:19:25,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/763fc6f7b5f04c6491b82454b048bb66 is 50, key is test_row_0/C:col10/1731421163378/Put/seqid=0 2024-11-12T14:19:25,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742467_1643 (size=12201) 2024-11-12T14:19:25,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=257 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/763fc6f7b5f04c6491b82454b048bb66 2024-11-12T14:19:25,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/e40542cdd9aa45e8b5b52fb789ff630c as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/e40542cdd9aa45e8b5b52fb789ff630c 2024-11-12T14:19:25,553 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:25,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421225550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:25,554 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_1' in e1aa16d140ed3f62a95c849e7bf6cc7a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:19:25,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:25,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:25,555 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:25,556 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/e40542cdd9aa45e8b5b52fb789ff630c, entries=150, sequenceid=257, filesize=30.4 K 2024-11-12T14:19:25,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/ab5cbd9d6da14b5c99285c65f36bd5d4 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/ab5cbd9d6da14b5c99285c65f36bd5d4 2024-11-12T14:19:25,560 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/ab5cbd9d6da14b5c99285c65f36bd5d4, entries=150, sequenceid=257, filesize=11.9 K 2024-11-12T14:19:25,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/763fc6f7b5f04c6491b82454b048bb66 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/763fc6f7b5f04c6491b82454b048bb66 2024-11-12T14:19:25,565 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/763fc6f7b5f04c6491b82454b048bb66, entries=150, sequenceid=257, filesize=11.9 K 2024-11-12T14:19:25,565 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e1aa16d140ed3f62a95c849e7bf6cc7a in 1519ms, sequenceid=257, compaction requested=false 2024-11-12T14:19:25,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:25,643 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-12T14:19:25,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:25,644 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing e1aa16d140ed3f62a95c849e7bf6cc7a 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T14:19:25,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=A 2024-11-12T14:19:25,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:25,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=B 2024-11-12T14:19:25,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:25,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=C 2024-11-12T14:19:25,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:25,654 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411129ee4dcd58778494ca43dba94b64f81e1_e1aa16d140ed3f62a95c849e7bf6cc7a is 50, key is test_row_0/A:col10/1731421164102/Put/seqid=0 2024-11-12T14:19:25,661 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742468_1644 (size=12454) 2024-11-12T14:19:25,663 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:19:25,667 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411129ee4dcd58778494ca43dba94b64f81e1_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129ee4dcd58778494ca43dba94b64f81e1_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:25,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/cb21b690e92b4100925a2eec83c0dd3a, store: [table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:25,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/cb21b690e92b4100925a2eec83c0dd3a is 175, key is test_row_0/A:col10/1731421164102/Put/seqid=0 2024-11-12T14:19:25,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742469_1645 (size=31255) 2024-11-12T14:19:25,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-12T14:19:26,086 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=274, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/cb21b690e92b4100925a2eec83c0dd3a 2024-11-12T14:19:26,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/406560d39dc44b2da671903b3f338eb7 is 50, key is test_row_0/B:col10/1731421164102/Put/seqid=0 2024-11-12T14:19:26,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742470_1646 (size=12301) 2024-11-12T14:19:26,094 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/406560d39dc44b2da671903b3f338eb7 2024-11-12T14:19:26,102 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/db9f2783f99445458c5fbeed4e25f3e0 is 50, key is test_row_0/C:col10/1731421164102/Put/seqid=0 2024-11-12T14:19:26,106 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742471_1647 (size=12301) 2024-11-12T14:19:26,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:26,304 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:26,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-12T14:19:26,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:26,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421226365, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:26,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:26,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421226366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:26,370 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:26,370 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:26,370 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:26,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:26,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:26,377 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:26,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:26,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421226480, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:26,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:26,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421226481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:26,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:26,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:26,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:26,487 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:26,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:26,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:26,507 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/db9f2783f99445458c5fbeed4e25f3e0 2024-11-12T14:19:26,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/cb21b690e92b4100925a2eec83c0dd3a as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/cb21b690e92b4100925a2eec83c0dd3a 2024-11-12T14:19:26,574 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/cb21b690e92b4100925a2eec83c0dd3a, entries=150, sequenceid=274, filesize=30.5 K 2024-11-12T14:19:26,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/406560d39dc44b2da671903b3f338eb7 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/406560d39dc44b2da671903b3f338eb7 2024-11-12T14:19:26,590 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/406560d39dc44b2da671903b3f338eb7, entries=150, sequenceid=274, filesize=12.0 K 2024-11-12T14:19:26,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/db9f2783f99445458c5fbeed4e25f3e0 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/db9f2783f99445458c5fbeed4e25f3e0 2024-11-12T14:19:26,597 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/db9f2783f99445458c5fbeed4e25f3e0, entries=150, sequenceid=274, filesize=12.0 K 2024-11-12T14:19:26,598 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=100.63 KB/103050 for e1aa16d140ed3f62a95c849e7bf6cc7a in 954ms, sequenceid=274, compaction requested=true 2024-11-12T14:19:26,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:26,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:26,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-12T14:19:26,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-12T14:19:26,601 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-12T14:19:26,601 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4160 sec 2024-11-12T14:19:26,603 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 1.4220 sec 2024-11-12T14:19:26,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:26,704 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1aa16d140ed3f62a95c849e7bf6cc7a 3/3 column families, dataSize=107.34 KB heapSize=319.50 KB 2024-11-12T14:19:26,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=A 2024-11-12T14:19:26,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:26,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=B 2024-11-12T14:19:26,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:26,704 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=C 2024-11-12T14:19:26,705 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:26,752 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111279a1cf6eb0bc4078ae7e3d9c74ac88db_e1aa16d140ed3f62a95c849e7bf6cc7a is 50, key is test_row_0/A:col10/1731421166348/Put/seqid=0 2024-11-12T14:19:26,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:26,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421226778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:26,790 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:26,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:26,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421226789, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:26,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:26,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:26,796 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:26,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742472_1648 (size=12454) 2024-11-12T14:19:26,801 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:19:26,807 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111279a1cf6eb0bc4078ae7e3d9c74ac88db_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111279a1cf6eb0bc4078ae7e3d9c74ac88db_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:26,808 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/1b0438a977114e7daeeb59c085728238, store: [table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:26,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/1b0438a977114e7daeeb59c085728238 is 175, key is test_row_0/A:col10/1731421166348/Put/seqid=0 2024-11-12T14:19:26,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742473_1649 (size=31255) 2024-11-12T14:19:26,822 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=293, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/1b0438a977114e7daeeb59c085728238 2024-11-12T14:19:26,830 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/133cb7010c6045da85da80a1f838e424 is 50, key is test_row_0/B:col10/1731421166348/Put/seqid=0 2024-11-12T14:19:26,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742474_1650 (size=12301) 2024-11-12T14:19:26,906 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:26,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421226900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:26,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:26,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:26,906 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:26,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:26,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421226901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:26,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:26,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:26,911 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:27,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421227119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:27,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:27,123 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:27,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:27,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:27,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421227119, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:27,123 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:27,123 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:27,123 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:27,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/133cb7010c6045da85da80a1f838e424 2024-11-12T14:19:27,273 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/54c68c3feffc45b4aea3a70462851016 is 50, key is test_row_0/C:col10/1731421166348/Put/seqid=0 2024-11-12T14:19:27,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742475_1651 (size=12301) 2024-11-12T14:19:27,298 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=293 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/54c68c3feffc45b4aea3a70462851016 2024-11-12T14:19:27,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/1b0438a977114e7daeeb59c085728238 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/1b0438a977114e7daeeb59c085728238 2024-11-12T14:19:27,305 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/1b0438a977114e7daeeb59c085728238, entries=150, sequenceid=293, filesize=30.5 K 2024-11-12T14:19:27,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/133cb7010c6045da85da80a1f838e424 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/133cb7010c6045da85da80a1f838e424 2024-11-12T14:19:27,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/133cb7010c6045da85da80a1f838e424, entries=150, sequenceid=293, filesize=12.0 K 2024-11-12T14:19:27,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/54c68c3feffc45b4aea3a70462851016 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/54c68c3feffc45b4aea3a70462851016 2024-11-12T14:19:27,319 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/54c68c3feffc45b4aea3a70462851016, entries=150, sequenceid=293, filesize=12.0 K 2024-11-12T14:19:27,320 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~319.45 KB/327120, currentSize=73.80 KB/75570 for e1aa16d140ed3f62a95c849e7bf6cc7a in 616ms, sequenceid=293, compaction requested=true 2024-11-12T14:19:27,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:27,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1aa16d140ed3f62a95c849e7bf6cc7a:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:19:27,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:27,320 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:19:27,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1aa16d140ed3f62a95c849e7bf6cc7a:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:19:27,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:27,320 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:19:27,320 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1aa16d140ed3f62a95c849e7bf6cc7a:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:19:27,321 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:19:27,321 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 125350 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:19:27,321 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): e1aa16d140ed3f62a95c849e7bf6cc7a/A is initiating minor compaction (all files) 2024-11-12T14:19:27,322 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1aa16d140ed3f62a95c849e7bf6cc7a/A in TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:27,322 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/454275c5db4b440eae27417a788332c5, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/e40542cdd9aa45e8b5b52fb789ff630c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/cb21b690e92b4100925a2eec83c0dd3a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/1b0438a977114e7daeeb59c085728238] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp, totalSize=122.4 K 2024-11-12T14:19:27,322 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:27,322 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. files: [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/454275c5db4b440eae27417a788332c5, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/e40542cdd9aa45e8b5b52fb789ff630c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/cb21b690e92b4100925a2eec83c0dd3a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/1b0438a977114e7daeeb59c085728238] 2024-11-12T14:19:27,322 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49534 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:19:27,322 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): e1aa16d140ed3f62a95c849e7bf6cc7a/B is initiating minor compaction (all files) 2024-11-12T14:19:27,322 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1aa16d140ed3f62a95c849e7bf6cc7a/B in TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:27,322 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/96365ebadcf046978831bbf67cccd0ed, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/ab5cbd9d6da14b5c99285c65f36bd5d4, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/406560d39dc44b2da671903b3f338eb7, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/133cb7010c6045da85da80a1f838e424] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp, totalSize=48.4 K 2024-11-12T14:19:27,322 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 454275c5db4b440eae27417a788332c5, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731421162721 2024-11-12T14:19:27,322 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 96365ebadcf046978831bbf67cccd0ed, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731421162721 2024-11-12T14:19:27,323 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting e40542cdd9aa45e8b5b52fb789ff630c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731421163365 2024-11-12T14:19:27,323 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting ab5cbd9d6da14b5c99285c65f36bd5d4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731421163365 2024-11-12T14:19:27,323 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 406560d39dc44b2da671903b3f338eb7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731421164102 2024-11-12T14:19:27,323 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting cb21b690e92b4100925a2eec83c0dd3a, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731421164102 2024-11-12T14:19:27,323 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 133cb7010c6045da85da80a1f838e424, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731421166348 2024-11-12T14:19:27,323 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b0438a977114e7daeeb59c085728238, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731421166348 2024-11-12T14:19:27,337 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1aa16d140ed3f62a95c849e7bf6cc7a#B#compaction#549 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:27,338 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/b2d47860dbed4543be0b1f203407546f is 50, key is test_row_0/B:col10/1731421166348/Put/seqid=0 2024-11-12T14:19:27,339 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:27,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-12T14:19:27,339 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:19:27,340 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:19:27,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-12T14:19:27,344 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:19:27,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T14:19:27,345 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:19:27,345 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:19:27,357 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411127490a1a326f546a3bbd15ac29947cd67_e1aa16d140ed3f62a95c849e7bf6cc7a store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:27,359 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411127490a1a326f546a3bbd15ac29947cd67_e1aa16d140ed3f62a95c849e7bf6cc7a, store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:27,360 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411127490a1a326f546a3bbd15ac29947cd67_e1aa16d140ed3f62a95c849e7bf6cc7a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:27,361 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742476_1652 (size=13017) 2024-11-12T14:19:27,388 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742477_1653 (size=4469) 2024-11-12T14:19:27,389 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1aa16d140ed3f62a95c849e7bf6cc7a#A#compaction#550 average throughput is 0.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:27,389 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/96ea226a39c4427fa9c8b164313961f4 is 175, key is test_row_0/A:col10/1731421166348/Put/seqid=0 2024-11-12T14:19:27,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742478_1654 (size=31971) 2024-11-12T14:19:27,411 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/96ea226a39c4427fa9c8b164313961f4 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/96ea226a39c4427fa9c8b164313961f4 2024-11-12T14:19:27,422 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1aa16d140ed3f62a95c849e7bf6cc7a/A of e1aa16d140ed3f62a95c849e7bf6cc7a into 96ea226a39c4427fa9c8b164313961f4(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:19:27,422 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:27,422 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., storeName=e1aa16d140ed3f62a95c849e7bf6cc7a/A, priority=12, startTime=1731421167320; duration=0sec 2024-11-12T14:19:27,423 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:19:27,423 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1aa16d140ed3f62a95c849e7bf6cc7a:A 2024-11-12T14:19:27,423 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:19:27,424 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49534 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:19:27,424 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): e1aa16d140ed3f62a95c849e7bf6cc7a/C is initiating minor compaction (all files) 2024-11-12T14:19:27,424 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1aa16d140ed3f62a95c849e7bf6cc7a/C in TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:27,424 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/35df06708ba6438b91692f9325cea0e6, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/763fc6f7b5f04c6491b82454b048bb66, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/db9f2783f99445458c5fbeed4e25f3e0, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/54c68c3feffc45b4aea3a70462851016] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp, totalSize=48.4 K 2024-11-12T14:19:27,424 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35df06708ba6438b91692f9325cea0e6, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1731421162721 2024-11-12T14:19:27,424 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 763fc6f7b5f04c6491b82454b048bb66, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=257, earliestPutTs=1731421163365 2024-11-12T14:19:27,426 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting db9f2783f99445458c5fbeed4e25f3e0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1731421164102 2024-11-12T14:19:27,426 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 54c68c3feffc45b4aea3a70462851016, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731421166348 2024-11-12T14:19:27,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:27,436 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1aa16d140ed3f62a95c849e7bf6cc7a 3/3 column families, dataSize=80.51 KB heapSize=239.81 KB 2024-11-12T14:19:27,436 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=A 2024-11-12T14:19:27,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:27,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=B 2024-11-12T14:19:27,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:27,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=C 2024-11-12T14:19:27,437 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:27,447 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1aa16d140ed3f62a95c849e7bf6cc7a#C#compaction#551 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:27,447 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/4753b75e3183448c8344955753ee8669 is 50, key is test_row_0/C:col10/1731421166348/Put/seqid=0 2024-11-12T14:19:27,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T14:19:27,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112035812fa48264f7387f0e04b32e659e0_e1aa16d140ed3f62a95c849e7bf6cc7a is 50, key is test_row_0/A:col10/1731421167434/Put/seqid=0 2024-11-12T14:19:27,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742479_1655 (size=13017) 2024-11-12T14:19:27,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T14:19:27,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:27,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:27,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:27,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:27,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:27,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:27,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742480_1656 (size=17534) 2024-11-12T14:19:27,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:27,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421227527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:27,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:27,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421227528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:27,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:27,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:27,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:27,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:27,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:27,535 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:27,575 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:27,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421227572, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:27,575 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_1' in e1aa16d140ed3f62a95c849e7bf6cc7a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:19:27,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:27,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:27,576 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:27,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:27,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421227639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:27,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:27,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:27,642 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:27,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:27,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421227644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:27,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:27,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:27,652 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:27,655 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T14:19:27,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:27,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:27,655 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:27,655 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:27,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:27,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:27,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T14:19:27,770 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/b2d47860dbed4543be0b1f203407546f as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/b2d47860dbed4543be0b1f203407546f 2024-11-12T14:19:27,775 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1aa16d140ed3f62a95c849e7bf6cc7a/B of e1aa16d140ed3f62a95c849e7bf6cc7a into b2d47860dbed4543be0b1f203407546f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:19:27,775 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:27,775 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., storeName=e1aa16d140ed3f62a95c849e7bf6cc7a/B, priority=12, startTime=1731421167320; duration=0sec 2024-11-12T14:19:27,775 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:27,775 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1aa16d140ed3f62a95c849e7bf6cc7a:B 2024-11-12T14:19:27,808 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T14:19:27,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:27,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:27,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:27,808 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:27,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:27,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:27,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:27,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421227849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:27,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:27,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:27,854 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:27,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:27,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421227859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:27,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:27,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:27,863 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:27,881 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/4753b75e3183448c8344955753ee8669 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/4753b75e3183448c8344955753ee8669 2024-11-12T14:19:27,898 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1aa16d140ed3f62a95c849e7bf6cc7a/C of e1aa16d140ed3f62a95c849e7bf6cc7a into 4753b75e3183448c8344955753ee8669(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:19:27,898 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:27,899 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., storeName=e1aa16d140ed3f62a95c849e7bf6cc7a/C, priority=12, startTime=1731421167320; duration=0sec 2024-11-12T14:19:27,899 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:27,899 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1aa16d140ed3f62a95c849e7bf6cc7a:C 2024-11-12T14:19:27,914 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:19:27,925 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112035812fa48264f7387f0e04b32e659e0_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112035812fa48264f7387f0e04b32e659e0_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:27,926 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/3bd83b4f04d3430eba7bad04705e414b, store: [table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:27,927 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/3bd83b4f04d3430eba7bad04705e414b is 175, key is test_row_0/A:col10/1731421167434/Put/seqid=0 2024-11-12T14:19:27,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742481_1657 (size=48639) 2024-11-12T14:19:27,965 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T14:19:27,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:27,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:27,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:27,965 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:27,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:27,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:27,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T14:19:28,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T14:19:28,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:28,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:28,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:28,121 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:28,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:28,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:28,163 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:28,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421228160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:28,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:28,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:28,164 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:28,170 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:28,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421228169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:28,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:28,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:28,170 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:28,279 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T14:19:28,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:28,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:28,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:28,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:28,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:28,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:28,363 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=310, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/3bd83b4f04d3430eba7bad04705e414b 2024-11-12T14:19:28,395 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/8bd0cbee31fb4f52a5a3f8982d63babc is 50, key is test_row_0/B:col10/1731421167434/Put/seqid=0 2024-11-12T14:19:28,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T14:19:28,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:28,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:28,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:28,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:28,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:28,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:28,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742482_1658 (size=12301) 2024-11-12T14:19:28,457 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/8bd0cbee31fb4f52a5a3f8982d63babc 2024-11-12T14:19:28,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T14:19:28,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/3c66a12905d44fd98c7be3c76e0448d0 is 50, key is test_row_0/C:col10/1731421167434/Put/seqid=0 2024-11-12T14:19:28,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742483_1659 (size=12301) 2024-11-12T14:19:28,548 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/3c66a12905d44fd98c7be3c76e0448d0 2024-11-12T14:19:28,571 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/3bd83b4f04d3430eba7bad04705e414b as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/3bd83b4f04d3430eba7bad04705e414b 2024-11-12T14:19:28,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T14:19:28,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:28,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:28,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:28,596 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:28,596 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:28,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:28,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/3bd83b4f04d3430eba7bad04705e414b, entries=250, sequenceid=310, filesize=47.5 K 2024-11-12T14:19:28,617 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/8bd0cbee31fb4f52a5a3f8982d63babc as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/8bd0cbee31fb4f52a5a3f8982d63babc 2024-11-12T14:19:28,640 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/8bd0cbee31fb4f52a5a3f8982d63babc, entries=150, sequenceid=310, filesize=12.0 K 2024-11-12T14:19:28,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/3c66a12905d44fd98c7be3c76e0448d0 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/3c66a12905d44fd98c7be3c76e0448d0 2024-11-12T14:19:28,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/3c66a12905d44fd98c7be3c76e0448d0, entries=150, sequenceid=310, filesize=12.0 K 2024-11-12T14:19:28,654 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~259.69 KB/265920, currentSize=93.93 KB/96180 for e1aa16d140ed3f62a95c849e7bf6cc7a in 1218ms, sequenceid=310, compaction requested=false 2024-11-12T14:19:28,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:28,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:28,678 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1aa16d140ed3f62a95c849e7bf6cc7a 3/3 column families, dataSize=100.63 KB heapSize=299.58 KB 2024-11-12T14:19:28,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=A 2024-11-12T14:19:28,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:28,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=B 2024-11-12T14:19:28,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:28,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=C 2024-11-12T14:19:28,678 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:28,710 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c0317c9b8d7447efa54a7a517dfe40c2_e1aa16d140ed3f62a95c849e7bf6cc7a is 50, key is test_row_0/A:col10/1731421167525/Put/seqid=0 2024-11-12T14:19:28,724 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:28,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421228719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:28,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:28,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:28,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:28,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:28,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421228723, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:28,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:28,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:28,728 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:28,752 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T14:19:28,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:28,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:28,752 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:28,753 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:28,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:28,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:28,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742484_1660 (size=14994) 2024-11-12T14:19:28,759 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:19:28,764 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112c0317c9b8d7447efa54a7a517dfe40c2_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c0317c9b8d7447efa54a7a517dfe40c2_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:28,764 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/c7b1cefc507544938dcf3980422318ec, store: [table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:28,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/c7b1cefc507544938dcf3980422318ec is 175, key is test_row_0/A:col10/1731421167525/Put/seqid=0 2024-11-12T14:19:28,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742485_1661 (size=39949) 2024-11-12T14:19:28,832 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:28,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421228829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:28,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:28,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:28,833 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:28,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:28,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421228839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:28,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:28,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:28,843 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:28,906 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T14:19:28,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:28,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:28,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:28,906 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:28,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:28,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:28,921 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-12T14:19:29,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:29,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43768 deadline: 1731421229029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:29,031 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_1' in e1aa16d140ed3f62a95c849e7bf6cc7a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8195 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:19:29,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:29,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:29,032 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:29,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:29,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421229039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:29,043 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:29,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:29,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:29,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:29,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421229049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:29,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:29,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:29,053 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:29,062 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T14:19:29,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:29,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:29,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:29,062 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:29,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:29,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:29,073 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:29,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43790 deadline: 1731421229069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:29,073 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_0' in e1aa16d140ed3f62a95c849e7bf6cc7a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:19:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:29,074 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:29,202 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=330, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/c7b1cefc507544938dcf3980422318ec 2024-11-12T14:19:29,214 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T14:19:29,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:29,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:29,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:29,215 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:29,215 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:29,216 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/936c949bd4464f37955092e6715a4c65 is 50, key is test_row_0/B:col10/1731421167525/Put/seqid=0 2024-11-12T14:19:29,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742486_1662 (size=12301) 2024-11-12T14:19:29,352 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:29,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421229352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:29,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:29,352 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:29,353 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:29,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:29,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421229359, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:29,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:29,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:29,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:29,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T14:19:29,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:29,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:29,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:29,368 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:29,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:29,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:29,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T14:19:29,524 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T14:19:29,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:29,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:29,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:29,525 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:29,525 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:29,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:29,658 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/936c949bd4464f37955092e6715a4c65 2024-11-12T14:19:29,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T14:19:29,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:29,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:29,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:29,684 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:29,684 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:29,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:29,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/da0ed1b128134abe8ef7ebdf05e71774 is 50, key is test_row_0/C:col10/1731421167525/Put/seqid=0 2024-11-12T14:19:29,745 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742487_1663 (size=12301) 2024-11-12T14:19:29,749 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/da0ed1b128134abe8ef7ebdf05e71774 2024-11-12T14:19:29,778 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/c7b1cefc507544938dcf3980422318ec as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/c7b1cefc507544938dcf3980422318ec 2024-11-12T14:19:29,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/c7b1cefc507544938dcf3980422318ec, entries=200, sequenceid=330, filesize=39.0 K 2024-11-12T14:19:29,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/936c949bd4464f37955092e6715a4c65 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/936c949bd4464f37955092e6715a4c65 2024-11-12T14:19:29,795 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/936c949bd4464f37955092e6715a4c65, entries=150, sequenceid=330, filesize=12.0 K 2024-11-12T14:19:29,796 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/da0ed1b128134abe8ef7ebdf05e71774 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/da0ed1b128134abe8ef7ebdf05e71774 2024-11-12T14:19:29,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/da0ed1b128134abe8ef7ebdf05e71774, entries=150, sequenceid=330, filesize=12.0 K 2024-11-12T14:19:29,802 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~299.53 KB/306720, currentSize=73.80 KB/75570 for e1aa16d140ed3f62a95c849e7bf6cc7a in 1124ms, sequenceid=330, compaction requested=true 2024-11-12T14:19:29,802 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:29,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1aa16d140ed3f62a95c849e7bf6cc7a:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:19:29,802 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:29,802 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:19:29,802 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:19:29,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1aa16d140ed3f62a95c849e7bf6cc7a:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:19:29,803 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:29,803 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 120559 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:19:29,803 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): e1aa16d140ed3f62a95c849e7bf6cc7a/A is initiating minor compaction (all files) 2024-11-12T14:19:29,803 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:19:29,803 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): e1aa16d140ed3f62a95c849e7bf6cc7a/B is initiating minor compaction (all files) 2024-11-12T14:19:29,803 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1aa16d140ed3f62a95c849e7bf6cc7a/A in TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:29,803 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1aa16d140ed3f62a95c849e7bf6cc7a/B in TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:29,803 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/96ea226a39c4427fa9c8b164313961f4, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/3bd83b4f04d3430eba7bad04705e414b, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/c7b1cefc507544938dcf3980422318ec] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp, totalSize=117.7 K 2024-11-12T14:19:29,803 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/b2d47860dbed4543be0b1f203407546f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/8bd0cbee31fb4f52a5a3f8982d63babc, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/936c949bd4464f37955092e6715a4c65] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp, totalSize=36.7 K 2024-11-12T14:19:29,803 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:29,803 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. files: [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/96ea226a39c4427fa9c8b164313961f4, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/3bd83b4f04d3430eba7bad04705e414b, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/c7b1cefc507544938dcf3980422318ec] 2024-11-12T14:19:29,803 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting b2d47860dbed4543be0b1f203407546f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731421166348 2024-11-12T14:19:29,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1aa16d140ed3f62a95c849e7bf6cc7a:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:19:29,804 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:19:29,805 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 8bd0cbee31fb4f52a5a3f8982d63babc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731421166777 2024-11-12T14:19:29,805 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96ea226a39c4427fa9c8b164313961f4, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731421166348 2024-11-12T14:19:29,805 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 936c949bd4464f37955092e6715a4c65, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731421167525 2024-11-12T14:19:29,806 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3bd83b4f04d3430eba7bad04705e414b, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731421166709 2024-11-12T14:19:29,806 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting c7b1cefc507544938dcf3980422318ec, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731421167517 2024-11-12T14:19:29,822 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1aa16d140ed3f62a95c849e7bf6cc7a#B#compaction#558 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:29,822 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/3f890509eda9499887e09a402178e2b9 is 50, key is test_row_0/B:col10/1731421167525/Put/seqid=0 2024-11-12T14:19:29,839 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:29,840 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-12T14:19:29,840 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:29,841 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing e1aa16d140ed3f62a95c849e7bf6cc7a 3/3 column families, dataSize=73.80 KB heapSize=219.89 KB 2024-11-12T14:19:29,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=A 2024-11-12T14:19:29,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:29,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=B 2024-11-12T14:19:29,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:29,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=C 2024-11-12T14:19:29,841 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:29,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742488_1664 (size=13119) 2024-11-12T14:19:29,849 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/3f890509eda9499887e09a402178e2b9 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/3f890509eda9499887e09a402178e2b9 2024-11-12T14:19:29,861 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411122849ebf6603e4edd968ba8dac4f3dfaa_e1aa16d140ed3f62a95c849e7bf6cc7a store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:29,864 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411122849ebf6603e4edd968ba8dac4f3dfaa_e1aa16d140ed3f62a95c849e7bf6cc7a, store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:29,864 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411122849ebf6603e4edd968ba8dac4f3dfaa_e1aa16d140ed3f62a95c849e7bf6cc7a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:29,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:29,866 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:29,870 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1aa16d140ed3f62a95c849e7bf6cc7a/B of e1aa16d140ed3f62a95c849e7bf6cc7a into 3f890509eda9499887e09a402178e2b9(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:19:29,870 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:29,870 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., storeName=e1aa16d140ed3f62a95c849e7bf6cc7a/B, priority=13, startTime=1731421169802; duration=0sec 2024-11-12T14:19:29,870 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:19:29,870 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1aa16d140ed3f62a95c849e7bf6cc7a:B 2024-11-12T14:19:29,871 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:19:29,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e57639a7b7cc4ba48221e282b5b13acf_e1aa16d140ed3f62a95c849e7bf6cc7a is 50, key is test_row_0/A:col10/1731421168705/Put/seqid=0 2024-11-12T14:19:29,882 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:19:29,882 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): e1aa16d140ed3f62a95c849e7bf6cc7a/C is initiating minor compaction (all files) 2024-11-12T14:19:29,882 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1aa16d140ed3f62a95c849e7bf6cc7a/C in TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:29,882 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/4753b75e3183448c8344955753ee8669, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/3c66a12905d44fd98c7be3c76e0448d0, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/da0ed1b128134abe8ef7ebdf05e71774] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp, totalSize=36.7 K 2024-11-12T14:19:29,882 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 4753b75e3183448c8344955753ee8669, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=293, earliestPutTs=1731421166348 2024-11-12T14:19:29,883 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c66a12905d44fd98c7be3c76e0448d0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731421166777 2024-11-12T14:19:29,883 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting da0ed1b128134abe8ef7ebdf05e71774, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731421167525 2024-11-12T14:19:29,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742489_1665 (size=4469) 2024-11-12T14:19:29,899 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1aa16d140ed3f62a95c849e7bf6cc7a#A#compaction#559 average throughput is 0.41 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:29,899 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/5cc9b7d4e41b42dda7f3c05a7ba3142a is 175, key is test_row_0/A:col10/1731421167525/Put/seqid=0 2024-11-12T14:19:29,915 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1aa16d140ed3f62a95c849e7bf6cc7a#C#compaction#561 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:29,916 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/ca8cd0ce39cd4b6db0a4c6cfbb758cae is 50, key is test_row_0/C:col10/1731421167525/Put/seqid=0 2024-11-12T14:19:29,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742490_1666 (size=12454) 2024-11-12T14:19:29,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:19:29,920 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112e57639a7b7cc4ba48221e282b5b13acf_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e57639a7b7cc4ba48221e282b5b13acf_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:29,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/9d9873b1afd54278af246f3fdd0b3414, store: [table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:29,933 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/9d9873b1afd54278af246f3fdd0b3414 is 175, key is test_row_0/A:col10/1731421168705/Put/seqid=0 2024-11-12T14:19:29,939 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742491_1667 (size=32073) 2024-11-12T14:19:29,946 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/5cc9b7d4e41b42dda7f3c05a7ba3142a as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/5cc9b7d4e41b42dda7f3c05a7ba3142a 2024-11-12T14:19:29,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:29,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421229942, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:29,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:29,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:29,950 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:29,951 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1aa16d140ed3f62a95c849e7bf6cc7a/A of e1aa16d140ed3f62a95c849e7bf6cc7a into 5cc9b7d4e41b42dda7f3c05a7ba3142a(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:19:29,951 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:29,951 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., storeName=e1aa16d140ed3f62a95c849e7bf6cc7a/A, priority=13, startTime=1731421169802; duration=0sec 2024-11-12T14:19:29,951 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:29,951 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1aa16d140ed3f62a95c849e7bf6cc7a:A 2024-11-12T14:19:29,951 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:29,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421229949, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:29,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:29,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:29,952 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:29,961 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742492_1668 (size=13119) 2024-11-12T14:19:29,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742493_1669 (size=31255) 2024-11-12T14:19:29,981 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=344, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/9d9873b1afd54278af246f3fdd0b3414 2024-11-12T14:19:30,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/8618b20e54d64b9c80fd299e569f0f6d is 50, key is test_row_0/B:col10/1731421168705/Put/seqid=0 2024-11-12T14:19:30,041 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742494_1670 (size=12301) 2024-11-12T14:19:30,045 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/8618b20e54d64b9c80fd299e569f0f6d 2024-11-12T14:19:30,063 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:30,063 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:30,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421230059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:30,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421230059, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:30,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:30,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:30,064 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:30,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:30,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:30,065 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:30,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/261129df156b4f52883f1ef99ef23a4f is 50, key is test_row_0/C:col10/1731421168705/Put/seqid=0 2024-11-12T14:19:30,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742495_1671 (size=12301) 2024-11-12T14:19:30,130 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=344 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/261129df156b4f52883f1ef99ef23a4f 2024-11-12T14:19:30,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/9d9873b1afd54278af246f3fdd0b3414 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/9d9873b1afd54278af246f3fdd0b3414 2024-11-12T14:19:30,140 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/9d9873b1afd54278af246f3fdd0b3414, entries=150, sequenceid=344, filesize=30.5 K 2024-11-12T14:19:30,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/8618b20e54d64b9c80fd299e569f0f6d as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/8618b20e54d64b9c80fd299e569f0f6d 2024-11-12T14:19:30,146 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/8618b20e54d64b9c80fd299e569f0f6d, entries=150, sequenceid=344, filesize=12.0 K 2024-11-12T14:19:30,146 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/261129df156b4f52883f1ef99ef23a4f as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/261129df156b4f52883f1ef99ef23a4f 2024-11-12T14:19:30,151 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/261129df156b4f52883f1ef99ef23a4f, entries=150, sequenceid=344, filesize=12.0 K 2024-11-12T14:19:30,152 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~219.84 KB/225120, currentSize=107.34 KB/109920 for e1aa16d140ed3f62a95c849e7bf6cc7a in 311ms, sequenceid=344, compaction requested=false 2024-11-12T14:19:30,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:30,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:30,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-12T14:19:30,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-12T14:19:30,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-12T14:19:30,158 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8110 sec 2024-11-12T14:19:30,160 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.8180 sec 2024-11-12T14:19:30,275 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1aa16d140ed3f62a95c849e7bf6cc7a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T14:19:30,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:30,276 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=A 2024-11-12T14:19:30,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:19:30,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=B 2024-11-12T14:19:30,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:19:30,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=C 2024-11-12T14:19:30,277 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=2, new segment=null 2024-11-12T14:19:30,284 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411123dd83a604cc248cd9febf0539327ad20_e1aa16d140ed3f62a95c849e7bf6cc7a is 50, key is test_row_0/A:col10/1731421169939/Put/seqid=0 2024-11-12T14:19:30,293 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742496_1672 (size=17534) 2024-11-12T14:19:30,295 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:19:30,298 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411123dd83a604cc248cd9febf0539327ad20_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123dd83a604cc248cd9febf0539327ad20_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:30,298 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/df37ba2636db46c88ede0c8dd285dc39, store: [table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:30,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/df37ba2636db46c88ede0c8dd285dc39 is 175, key is test_row_0/A:col10/1731421169939/Put/seqid=0 2024-11-12T14:19:30,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:30,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421230296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:30,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:30,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:30,302 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:30,305 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:30,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421230300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:30,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:30,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:30,306 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:30,352 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742497_1673 (size=48639) 2024-11-12T14:19:30,356 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=367, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/df37ba2636db46c88ede0c8dd285dc39 2024-11-12T14:19:30,388 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/ca8cd0ce39cd4b6db0a4c6cfbb758cae as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/ca8cd0ce39cd4b6db0a4c6cfbb758cae 2024-11-12T14:19:30,408 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1aa16d140ed3f62a95c849e7bf6cc7a/C of e1aa16d140ed3f62a95c849e7bf6cc7a into ca8cd0ce39cd4b6db0a4c6cfbb758cae(size=12.8 K), total size for store is 24.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:19:30,408 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:30,408 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., storeName=e1aa16d140ed3f62a95c849e7bf6cc7a/C, priority=13, startTime=1731421169803; duration=0sec 2024-11-12T14:19:30,408 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:30,408 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1aa16d140ed3f62a95c849e7bf6cc7a:C 2024-11-12T14:19:30,413 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:30,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421230410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:30,414 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:30,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421230410, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:30,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:30,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:30,415 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:30,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:30,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:30,415 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:30,420 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/32c6a9a278dd48d5bd7ce1b9f8e7bd5e is 50, key is test_row_0/B:col10/1731421169939/Put/seqid=0 2024-11-12T14:19:30,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742498_1674 (size=12301) 2024-11-12T14:19:30,622 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:30,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421230620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:30,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:30,623 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:30,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:30,623 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:30,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421230620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:30,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:30,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:30,623 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:30,875 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/32c6a9a278dd48d5bd7ce1b9f8e7bd5e 2024-11-12T14:19:30,883 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/b080fa71f7de42d08bc81ed3f346e608 is 50, key is test_row_0/C:col10/1731421169939/Put/seqid=0 2024-11-12T14:19:30,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742499_1675 (size=12301) 2024-11-12T14:19:30,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:30,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421230929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:30,934 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:30,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:30,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:30,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421230929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:30,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:30,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:30,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:30,934 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:31,310 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/b080fa71f7de42d08bc81ed3f346e608 2024-11-12T14:19:31,315 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/df37ba2636db46c88ede0c8dd285dc39 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/df37ba2636db46c88ede0c8dd285dc39 2024-11-12T14:19:31,323 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/df37ba2636db46c88ede0c8dd285dc39, entries=250, sequenceid=367, filesize=47.5 K 2024-11-12T14:19:31,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/32c6a9a278dd48d5bd7ce1b9f8e7bd5e as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/32c6a9a278dd48d5bd7ce1b9f8e7bd5e 2024-11-12T14:19:31,332 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/32c6a9a278dd48d5bd7ce1b9f8e7bd5e, entries=150, sequenceid=367, filesize=12.0 K 2024-11-12T14:19:31,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/b080fa71f7de42d08bc81ed3f346e608 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/b080fa71f7de42d08bc81ed3f346e608 2024-11-12T14:19:31,338 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/b080fa71f7de42d08bc81ed3f346e608, entries=150, sequenceid=367, filesize=12.0 K 2024-11-12T14:19:31,339 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~360 KB/368640, currentSize=53.67 KB/54960 for e1aa16d140ed3f62a95c849e7bf6cc7a in 1064ms, sequenceid=367, compaction requested=true 2024-11-12T14:19:31,339 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:31,339 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:19:31,340 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111967 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:19:31,340 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): e1aa16d140ed3f62a95c849e7bf6cc7a/A is initiating minor compaction (all files) 2024-11-12T14:19:31,340 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1aa16d140ed3f62a95c849e7bf6cc7a/A in TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:31,340 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/5cc9b7d4e41b42dda7f3c05a7ba3142a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/9d9873b1afd54278af246f3fdd0b3414, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/df37ba2636db46c88ede0c8dd285dc39] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp, totalSize=109.3 K 2024-11-12T14:19:31,340 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:31,340 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. files: [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/5cc9b7d4e41b42dda7f3c05a7ba3142a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/9d9873b1afd54278af246f3fdd0b3414, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/df37ba2636db46c88ede0c8dd285dc39] 2024-11-12T14:19:31,341 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cc9b7d4e41b42dda7f3c05a7ba3142a, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731421167525 2024-11-12T14:19:31,341 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d9873b1afd54278af246f3fdd0b3414, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731421168705 2024-11-12T14:19:31,341 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting df37ba2636db46c88ede0c8dd285dc39, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731421169933 2024-11-12T14:19:31,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1aa16d140ed3f62a95c849e7bf6cc7a:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:19:31,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:31,346 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:19:31,347 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:19:31,347 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): e1aa16d140ed3f62a95c849e7bf6cc7a/B is initiating minor compaction (all files) 2024-11-12T14:19:31,347 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1aa16d140ed3f62a95c849e7bf6cc7a/B in TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:31,347 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/3f890509eda9499887e09a402178e2b9, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/8618b20e54d64b9c80fd299e569f0f6d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/32c6a9a278dd48d5bd7ce1b9f8e7bd5e] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp, totalSize=36.8 K 2024-11-12T14:19:31,347 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 3f890509eda9499887e09a402178e2b9, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731421167525 2024-11-12T14:19:31,348 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 8618b20e54d64b9c80fd299e569f0f6d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731421168705 2024-11-12T14:19:31,348 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 32c6a9a278dd48d5bd7ce1b9f8e7bd5e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731421169939 2024-11-12T14:19:31,351 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1aa16d140ed3f62a95c849e7bf6cc7a:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:19:31,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:31,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1aa16d140ed3f62a95c849e7bf6cc7a:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:19:31,352 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:19:31,353 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:31,357 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1aa16d140ed3f62a95c849e7bf6cc7a#B#compaction#568 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:31,357 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/208794e3a2704603808fae4c39776f88 is 50, key is test_row_0/B:col10/1731421169939/Put/seqid=0 2024-11-12T14:19:31,373 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111265ee1b75f3b14434a78560a4947cd642_e1aa16d140ed3f62a95c849e7bf6cc7a store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:31,375 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111265ee1b75f3b14434a78560a4947cd642_e1aa16d140ed3f62a95c849e7bf6cc7a, store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:31,375 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111265ee1b75f3b14434a78560a4947cd642_e1aa16d140ed3f62a95c849e7bf6cc7a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:31,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742500_1676 (size=13221) 2024-11-12T14:19:31,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:31,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1aa16d140ed3f62a95c849e7bf6cc7a 3/3 column families, dataSize=60.38 KB heapSize=180.05 KB 2024-11-12T14:19:31,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=A 2024-11-12T14:19:31,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:31,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=B 2024-11-12T14:19:31,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:31,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=C 2024-11-12T14:19:31,442 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:31,461 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111257f432a823ba47099132d06833221eaa_e1aa16d140ed3f62a95c849e7bf6cc7a is 50, key is test_row_0/A:col10/1731421171440/Put/seqid=0 2024-11-12T14:19:31,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742501_1677 (size=4469) 2024-11-12T14:19:31,524 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742502_1678 (size=14994) 2024-11-12T14:19:31,525 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:19:31,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-12T14:19:31,532 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:19:31,533 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-12T14:19:31,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-12T14:19:31,535 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-12T14:19:31,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-12T14:19:31,537 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-12T14:19:31,537 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-12T14:19:31,554 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111257f432a823ba47099132d06833221eaa_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111257f432a823ba47099132d06833221eaa_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:31,555 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/8b1d9f325e694da1a3ed381d49bc417b, store: [table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:31,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/8b1d9f325e694da1a3ed381d49bc417b is 175, key is test_row_0/A:col10/1731421171440/Put/seqid=0 2024-11-12T14:19:31,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:31,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421231563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:31,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:31,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:31,565 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:31,570 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:31,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421231568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:31,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:31,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:31,571 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:31,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742503_1679 (size=39949) 2024-11-12T14:19:31,625 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:31,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43770 deadline: 1731421231624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:31,626 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 2b6d221c5cde,42925,1731420990822 for 'test_row_1' in e1aa16d140ed3f62a95c849e7bf6cc7a of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor43.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] ... 32 more 2024-11-12T14:19:31,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:31,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:31,626 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:31,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-12T14:19:31,673 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:31,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421231669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:31,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:31,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:31,673 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:31,683 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:31,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421231679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:31,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:31,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:31,683 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:31,689 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-12T14:19:31,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:31,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:31,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:31,690 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:31,690 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:31,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:31,806 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/208794e3a2704603808fae4c39776f88 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/208794e3a2704603808fae4c39776f88 2024-11-12T14:19:31,810 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1aa16d140ed3f62a95c849e7bf6cc7a/B of e1aa16d140ed3f62a95c849e7bf6cc7a into 208794e3a2704603808fae4c39776f88(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:19:31,810 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:31,810 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., storeName=e1aa16d140ed3f62a95c849e7bf6cc7a/B, priority=13, startTime=1731421171346; duration=0sec 2024-11-12T14:19:31,810 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:19:31,810 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1aa16d140ed3f62a95c849e7bf6cc7a:B 2024-11-12T14:19:31,810 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-12T14:19:31,812 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-12T14:19:31,812 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): e1aa16d140ed3f62a95c849e7bf6cc7a/C is initiating minor compaction (all files) 2024-11-12T14:19:31,812 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1aa16d140ed3f62a95c849e7bf6cc7a/C in TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:31,812 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/ca8cd0ce39cd4b6db0a4c6cfbb758cae, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/261129df156b4f52883f1ef99ef23a4f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/b080fa71f7de42d08bc81ed3f346e608] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp, totalSize=36.8 K 2024-11-12T14:19:31,813 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting ca8cd0ce39cd4b6db0a4c6cfbb758cae, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1731421167525 2024-11-12T14:19:31,813 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 261129df156b4f52883f1ef99ef23a4f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=344, earliestPutTs=1731421168705 2024-11-12T14:19:31,813 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting b080fa71f7de42d08bc81ed3f346e608, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731421169939 2024-11-12T14:19:31,822 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1aa16d140ed3f62a95c849e7bf6cc7a#C#compaction#570 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:31,822 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/fa9b9f0e454648639f75e18107ad6676 is 50, key is test_row_0/C:col10/1731421169939/Put/seqid=0 2024-11-12T14:19:31,831 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742504_1680 (size=13221) 2024-11-12T14:19:31,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-12T14:19:31,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:31,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:31,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:31,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:31,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:31,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:31,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-12T14:19:31,873 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1aa16d140ed3f62a95c849e7bf6cc7a#A#compaction#567 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:31,874 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/4e5cfac9ad2644058c412693800fb444 is 175, key is test_row_0/A:col10/1731421169939/Put/seqid=0 2024-11-12T14:19:31,882 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:31,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421231879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:31,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:31,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:31,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:31,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742505_1681 (size=32175) 2024-11-12T14:19:31,893 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:31,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421231889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:31,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:31,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:31,893 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:31,984 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=380, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/8b1d9f325e694da1a3ed381d49bc417b 2024-11-12T14:19:31,990 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/c33c5de9ebce4ae0849c7bff5c49a422 is 50, key is test_row_0/B:col10/1731421171440/Put/seqid=0 2024-11-12T14:19:31,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-12T14:19:31,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:31,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:31,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:31,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:31,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:31,997 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742506_1682 (size=12301) 2024-11-12T14:19:31,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-12T14:19:31,998 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/c33c5de9ebce4ae0849c7bff5c49a422 2024-11-12T14:19:32,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/ab94fc0d69134181a77d5c148a844655 is 50, key is test_row_0/C:col10/1731421171440/Put/seqid=0 2024-11-12T14:19:32,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742507_1683 (size=12301) 2024-11-12T14:19:32,019 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=380 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/ab94fc0d69134181a77d5c148a844655 2024-11-12T14:19:32,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/8b1d9f325e694da1a3ed381d49bc417b as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/8b1d9f325e694da1a3ed381d49bc417b 2024-11-12T14:19:32,029 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/8b1d9f325e694da1a3ed381d49bc417b, entries=200, sequenceid=380, filesize=39.0 K 2024-11-12T14:19:32,030 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/c33c5de9ebce4ae0849c7bff5c49a422 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/c33c5de9ebce4ae0849c7bff5c49a422 2024-11-12T14:19:32,034 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/c33c5de9ebce4ae0849c7bff5c49a422, entries=150, sequenceid=380, filesize=12.0 K 2024-11-12T14:19:32,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/ab94fc0d69134181a77d5c148a844655 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/ab94fc0d69134181a77d5c148a844655 2024-11-12T14:19:32,039 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/ab94fc0d69134181a77d5c148a844655, entries=150, sequenceid=380, filesize=12.0 K 2024-11-12T14:19:32,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~180 KB/184320, currentSize=114.05 KB/116790 for e1aa16d140ed3f62a95c849e7bf6cc7a in 598ms, sequenceid=380, compaction requested=false 2024-11-12T14:19:32,040 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:32,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-12T14:19:32,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:32,150 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing e1aa16d140ed3f62a95c849e7bf6cc7a 3/3 column families, dataSize=114.05 KB heapSize=339.42 KB 2024-11-12T14:19:32,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=A 2024-11-12T14:19:32,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:32,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=B 2024-11-12T14:19:32,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:32,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=C 2024-11-12T14:19:32,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:32,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-12T14:19:32,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125422d87f509f47e5b5e6a35f03746ca5_e1aa16d140ed3f62a95c849e7bf6cc7a is 50, key is test_row_0/A:col10/1731421171563/Put/seqid=0 2024-11-12T14:19:32,177 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742508_1684 (size=12454) 2024-11-12T14:19:32,178 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:19:32,188 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411125422d87f509f47e5b5e6a35f03746ca5_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125422d87f509f47e5b5e6a35f03746ca5_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:32,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/3b09656d168d4164ba73dd630b011f43, store: [table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:32,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/3b09656d168d4164ba73dd630b011f43 is 175, key is test_row_0/A:col10/1731421171563/Put/seqid=0 2024-11-12T14:19:32,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:32,192 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. as already flushing 2024-11-12T14:19:32,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742509_1685 (size=31255) 2024-11-12T14:19:32,206 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=401, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/3b09656d168d4164ba73dd630b011f43 2024-11-12T14:19:32,212 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/1bc626d526de461dbf0184ba6c2159ec is 50, key is test_row_0/B:col10/1731421171563/Put/seqid=0 2024-11-12T14:19:32,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742510_1686 (size=12301) 2024-11-12T14:19:32,225 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/1bc626d526de461dbf0184ba6c2159ec 2024-11-12T14:19:32,226 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:32,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43776 deadline: 1731421232221, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:32,226 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:32,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:32,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:32,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/b162878efd4b411c9223a3b06dc3cce3 is 50, key is test_row_0/C:col10/1731421171563/Put/seqid=0 2024-11-12T14:19:32,235 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-3.0.0-beta-2-SNAPSHOT.jar:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:3.0.0-beta-2-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:3.0.0-beta-2-SNAPSHOT] 2024-11-12T14:19:32,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:43764 deadline: 1731421232227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:32,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 , the old value is region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:32,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-12T14:19:32,235 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., hostname=2b6d221c5cde,42925,1731420990822, seqNum=5 because the exception is null or not the one we care about 2024-11-12T14:19:32,241 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742511_1687 (size=12301) 2024-11-12T14:19:32,244 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/fa9b9f0e454648639f75e18107ad6676 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/fa9b9f0e454648639f75e18107ad6676 2024-11-12T14:19:32,247 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=401 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/b162878efd4b411c9223a3b06dc3cce3 2024-11-12T14:19:32,257 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1aa16d140ed3f62a95c849e7bf6cc7a/C of e1aa16d140ed3f62a95c849e7bf6cc7a into fa9b9f0e454648639f75e18107ad6676(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:19:32,257 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:32,257 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., storeName=e1aa16d140ed3f62a95c849e7bf6cc7a/C, priority=13, startTime=1731421171352; duration=0sec 2024-11-12T14:19:32,257 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:32,257 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1aa16d140ed3f62a95c849e7bf6cc7a:C 2024-11-12T14:19:32,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/3b09656d168d4164ba73dd630b011f43 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/3b09656d168d4164ba73dd630b011f43 2024-11-12T14:19:32,264 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/3b09656d168d4164ba73dd630b011f43, entries=150, sequenceid=401, filesize=30.5 K 2024-11-12T14:19:32,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/1bc626d526de461dbf0184ba6c2159ec as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/1bc626d526de461dbf0184ba6c2159ec 2024-11-12T14:19:32,269 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/1bc626d526de461dbf0184ba6c2159ec, entries=150, sequenceid=401, filesize=12.0 K 2024-11-12T14:19:32,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/b162878efd4b411c9223a3b06dc3cce3 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/b162878efd4b411c9223a3b06dc3cce3 2024-11-12T14:19:32,281 INFO [Thread-2535 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2535. 2024-11-12T14:19:32,281 DEBUG [Thread-2535 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:19:32,281 DEBUG [Thread-2535 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:32,281 DEBUG [Thread-2535 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:32,281 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:19:32,282 INFO [Thread-2543 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2543. 2024-11-12T14:19:32,282 DEBUG [Thread-2543 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:19:32,282 DEBUG [Thread-2543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:32,282 DEBUG [Thread-2543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:32,282 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:19:32,286 INFO [Thread-2539 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2539. 2024-11-12T14:19:32,286 DEBUG [Thread-2539 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:19:32,286 DEBUG [Thread-2539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:32,286 DEBUG [Thread-2539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:32,286 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:19:32,287 INFO [Thread-2541 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2541. 2024-11-12T14:19:32,287 DEBUG [Thread-2541 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:19:32,287 DEBUG [Thread-2541 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:32,287 DEBUG [Thread-2541 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:32,287 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:19:32,288 INFO [Thread-2537 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2537. 2024-11-12T14:19:32,288 DEBUG [Thread-2537 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:19:32,288 DEBUG [Thread-2537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:32,288 DEBUG [Thread-2537 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:32,291 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/4e5cfac9ad2644058c412693800fb444 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/4e5cfac9ad2644058c412693800fb444 2024-11-12T14:19:32,291 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:19:32,292 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/b162878efd4b411c9223a3b06dc3cce3, entries=150, sequenceid=401, filesize=12.0 K 2024-11-12T14:19:32,293 INFO [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~339.38 KB/347520, currentSize=60.38 KB/61830 for e1aa16d140ed3f62a95c849e7bf6cc7a in 143ms, sequenceid=401, compaction requested=true 2024-11-12T14:19:32,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:32,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:32,293 DEBUG [RS_FLUSH_OPERATIONS-regionserver/2b6d221c5cde:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-12T14:19:32,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-12T14:19:32,298 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-12T14:19:32,298 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 758 msec 2024-11-12T14:19:32,299 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in e1aa16d140ed3f62a95c849e7bf6cc7a/A of e1aa16d140ed3f62a95c849e7bf6cc7a into 4e5cfac9ad2644058c412693800fb444(size=31.4 K), total size for store is 101.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:19:32,299 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:32,299 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., storeName=e1aa16d140ed3f62a95c849e7bf6cc7a/A, priority=13, startTime=1731421171339; duration=0sec 2024-11-12T14:19:32,299 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:32,299 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1aa16d140ed3f62a95c849e7bf6cc7a:A 2024-11-12T14:19:32,300 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 766 msec 2024-11-12T14:19:32,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:32,331 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing e1aa16d140ed3f62a95c849e7bf6cc7a 3/3 column families, dataSize=67.09 KB heapSize=199.97 KB 2024-11-12T14:19:32,331 INFO [Thread-2532 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2532. 2024-11-12T14:19:32,331 DEBUG [Thread-2532 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:19:32,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=A 2024-11-12T14:19:32,331 DEBUG [Thread-2532 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:32,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:32,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=B 2024-11-12T14:19:32,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:32,331 DEBUG [Thread-2532 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:32,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=C 2024-11-12T14:19:32,331 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:32,331 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:19:32,336 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112be83a2823a8b4e1d807f1ba71011c4d8_e1aa16d140ed3f62a95c849e7bf6cc7a is 50, key is test_row_0/A:col10/1731421172225/Put/seqid=0 2024-11-12T14:19:32,340 INFO [Thread-2526 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2526. 2024-11-12T14:19:32,341 DEBUG [Thread-2526 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:19:32,341 DEBUG [Thread-2526 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:32,341 DEBUG [Thread-2526 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:32,341 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:19:32,348 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742512_1688 (size=12454) 2024-11-12T14:19:32,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-12T14:19:32,669 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:19:32,746 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:19:32,749 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112be83a2823a8b4e1d807f1ba71011c4d8_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112be83a2823a8b4e1d807f1ba71011c4d8_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:32,750 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/14978444336d4067a642ce314407ed14, store: [table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:32,750 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/14978444336d4067a642ce314407ed14 is 175, key is test_row_0/A:col10/1731421172225/Put/seqid=0 2024-11-12T14:19:32,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742513_1689 (size=31255) 2024-11-12T14:19:33,154 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=416, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/14978444336d4067a642ce314407ed14 2024-11-12T14:19:33,161 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/b780e2478f3e4f46ac9f6777c0b0370c is 50, key is test_row_0/B:col10/1731421172225/Put/seqid=0 2024-11-12T14:19:33,164 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742514_1690 (size=12301) 2024-11-12T14:19:33,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/b780e2478f3e4f46ac9f6777c0b0370c 2024-11-12T14:19:33,569 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/c17c39008f114ecca5a8fb9d75064bed is 50, key is test_row_0/C:col10/1731421172225/Put/seqid=0 2024-11-12T14:19:33,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742515_1691 (size=12301) 2024-11-12T14:19:33,972 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=416 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/c17c39008f114ecca5a8fb9d75064bed 2024-11-12T14:19:33,977 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/14978444336d4067a642ce314407ed14 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/14978444336d4067a642ce314407ed14 2024-11-12T14:19:33,994 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/14978444336d4067a642ce314407ed14, entries=150, sequenceid=416, filesize=30.5 K 2024-11-12T14:19:33,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/b780e2478f3e4f46ac9f6777c0b0370c as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/b780e2478f3e4f46ac9f6777c0b0370c 2024-11-12T14:19:33,999 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/b780e2478f3e4f46ac9f6777c0b0370c, entries=150, sequenceid=416, filesize=12.0 K 2024-11-12T14:19:34,000 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/c17c39008f114ecca5a8fb9d75064bed as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/c17c39008f114ecca5a8fb9d75064bed 2024-11-12T14:19:34,003 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/c17c39008f114ecca5a8fb9d75064bed, entries=150, sequenceid=416, filesize=12.0 K 2024-11-12T14:19:34,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~199.92 KB/204720, currentSize=6.71 KB/6870 for e1aa16d140ed3f62a95c849e7bf6cc7a in 1674ms, sequenceid=416, compaction requested=true 2024-11-12T14:19:34,004 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:34,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1aa16d140ed3f62a95c849e7bf6cc7a:A, priority=-2147483648, current under compaction store size is 1 2024-11-12T14:19:34,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:34,004 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:19:34,004 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1aa16d140ed3f62a95c849e7bf6cc7a:B, priority=-2147483648, current under compaction store size is 2 2024-11-12T14:19:34,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:34,005 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:19:34,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store e1aa16d140ed3f62a95c849e7bf6cc7a:C, priority=-2147483648, current under compaction store size is 3 2024-11-12T14:19:34,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:19:34,005 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 134634 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:19:34,005 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:19:34,005 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1541): e1aa16d140ed3f62a95c849e7bf6cc7a/A is initiating minor compaction (all files) 2024-11-12T14:19:34,005 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): e1aa16d140ed3f62a95c849e7bf6cc7a/B is initiating minor compaction (all files) 2024-11-12T14:19:34,005 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1aa16d140ed3f62a95c849e7bf6cc7a/A in TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:34,005 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1aa16d140ed3f62a95c849e7bf6cc7a/B in TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:34,006 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/4e5cfac9ad2644058c412693800fb444, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/8b1d9f325e694da1a3ed381d49bc417b, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/3b09656d168d4164ba73dd630b011f43, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/14978444336d4067a642ce314407ed14] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp, totalSize=131.5 K 2024-11-12T14:19:34,006 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/208794e3a2704603808fae4c39776f88, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/c33c5de9ebce4ae0849c7bff5c49a422, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/1bc626d526de461dbf0184ba6c2159ec, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/b780e2478f3e4f46ac9f6777c0b0370c] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp, totalSize=48.9 K 2024-11-12T14:19:34,006 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:34,006 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. files: [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/4e5cfac9ad2644058c412693800fb444, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/8b1d9f325e694da1a3ed381d49bc417b, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/3b09656d168d4164ba73dd630b011f43, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/14978444336d4067a642ce314407ed14] 2024-11-12T14:19:34,006 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 208794e3a2704603808fae4c39776f88, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731421169939 2024-11-12T14:19:34,006 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e5cfac9ad2644058c412693800fb444, keycount=150, bloomtype=ROW, size=31.4 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731421169939 2024-11-12T14:19:34,006 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting c33c5de9ebce4ae0849c7bff5c49a422, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731421170280 2024-11-12T14:19:34,006 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b1d9f325e694da1a3ed381d49bc417b, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731421170280 2024-11-12T14:19:34,006 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bc626d526de461dbf0184ba6c2159ec, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731421171530 2024-11-12T14:19:34,006 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b09656d168d4164ba73dd630b011f43, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731421171530 2024-11-12T14:19:34,006 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting b780e2478f3e4f46ac9f6777c0b0370c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731421172209 2024-11-12T14:19:34,007 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14978444336d4067a642ce314407ed14, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731421172209 2024-11-12T14:19:34,017 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1aa16d140ed3f62a95c849e7bf6cc7a#B#compaction#579 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:34,018 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/5db46b1e955b4fbeb9bf6667278a5825 is 50, key is test_row_0/B:col10/1731421172225/Put/seqid=0 2024-11-12T14:19:34,022 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:34,030 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111228c9952ae53d4cf398634827c75ad052_e1aa16d140ed3f62a95c849e7bf6cc7a store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:34,040 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111228c9952ae53d4cf398634827c75ad052_e1aa16d140ed3f62a95c849e7bf6cc7a, store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:34,040 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111228c9952ae53d4cf398634827c75ad052_e1aa16d140ed3f62a95c849e7bf6cc7a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:34,044 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742516_1692 (size=13357) 2024-11-12T14:19:34,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742517_1693 (size=4469) 2024-11-12T14:19:34,080 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1aa16d140ed3f62a95c849e7bf6cc7a#A#compaction#580 average throughput is 0.42 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:34,081 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/1647fcd9ed584ed68616622fcac81417 is 175, key is test_row_0/A:col10/1731421172225/Put/seqid=0 2024-11-12T14:19:34,082 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/5db46b1e955b4fbeb9bf6667278a5825 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/5db46b1e955b4fbeb9bf6667278a5825 2024-11-12T14:19:34,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742518_1694 (size=32311) 2024-11-12T14:19:34,106 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/1647fcd9ed584ed68616622fcac81417 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/1647fcd9ed584ed68616622fcac81417 2024-11-12T14:19:34,107 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1aa16d140ed3f62a95c849e7bf6cc7a/B of e1aa16d140ed3f62a95c849e7bf6cc7a into 5db46b1e955b4fbeb9bf6667278a5825(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:19:34,107 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:34,107 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., storeName=e1aa16d140ed3f62a95c849e7bf6cc7a/B, priority=12, startTime=1731421174004; duration=0sec 2024-11-12T14:19:34,107 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-12T14:19:34,107 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1aa16d140ed3f62a95c849e7bf6cc7a:B 2024-11-12T14:19:34,107 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-12T14:19:34,108 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50124 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-12T14:19:34,109 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1541): e1aa16d140ed3f62a95c849e7bf6cc7a/C is initiating minor compaction (all files) 2024-11-12T14:19:34,109 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of e1aa16d140ed3f62a95c849e7bf6cc7a/C in TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:34,109 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/fa9b9f0e454648639f75e18107ad6676, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/ab94fc0d69134181a77d5c148a844655, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/b162878efd4b411c9223a3b06dc3cce3, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/c17c39008f114ecca5a8fb9d75064bed] into tmpdir=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp, totalSize=48.9 K 2024-11-12T14:19:34,113 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting fa9b9f0e454648639f75e18107ad6676, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1731421169939 2024-11-12T14:19:34,116 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting ab94fc0d69134181a77d5c148a844655, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=380, earliestPutTs=1731421170280 2024-11-12T14:19:34,117 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting b162878efd4b411c9223a3b06dc3cce3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=401, earliestPutTs=1731421171530 2024-11-12T14:19:34,117 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting c17c39008f114ecca5a8fb9d75064bed, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=416, earliestPutTs=1731421172209 2024-11-12T14:19:34,118 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1aa16d140ed3f62a95c849e7bf6cc7a/A of e1aa16d140ed3f62a95c849e7bf6cc7a into 1647fcd9ed584ed68616622fcac81417(size=31.6 K), total size for store is 31.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:19:34,118 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:34,118 INFO [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., storeName=e1aa16d140ed3f62a95c849e7bf6cc7a/A, priority=12, startTime=1731421174004; duration=0sec 2024-11-12T14:19:34,118 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:34,118 DEBUG [RS:0;2b6d221c5cde:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1aa16d140ed3f62a95c849e7bf6cc7a:A 2024-11-12T14:19:34,145 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): e1aa16d140ed3f62a95c849e7bf6cc7a#C#compaction#581 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-12T14:19:34,145 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/b2a175111577431da9248f60355012aa is 50, key is test_row_0/C:col10/1731421172225/Put/seqid=0 2024-11-12T14:19:34,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742519_1695 (size=13357) 2024-11-12T14:19:34,576 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/b2a175111577431da9248f60355012aa as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/b2a175111577431da9248f60355012aa 2024-11-12T14:19:34,579 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in e1aa16d140ed3f62a95c849e7bf6cc7a/C of e1aa16d140ed3f62a95c849e7bf6cc7a into b2a175111577431da9248f60355012aa(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-12T14:19:34,579 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for e1aa16d140ed3f62a95c849e7bf6cc7a: 2024-11-12T14:19:34,579 INFO [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., storeName=e1aa16d140ed3f62a95c849e7bf6cc7a/C, priority=12, startTime=1731421174005; duration=0sec 2024-11-12T14:19:34,579 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-12T14:19:34,579 DEBUG [RS:0;2b6d221c5cde:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: e1aa16d140ed3f62a95c849e7bf6cc7a:C 2024-11-12T14:19:39,050 INFO [Thread-2524 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2524. 2024-11-12T14:19:39,051 DEBUG [Thread-2524 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:19:39,051 DEBUG [Thread-2524 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:39,051 DEBUG [Thread-2524 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:39,051 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:19:39,161 INFO [Thread-2530 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2530. 2024-11-12T14:19:39,161 DEBUG [Thread-2530 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:19:39,161 DEBUG [Thread-2530 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:39,161 DEBUG [Thread-2530 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:39,161 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:19:41,670 INFO [Thread-2528 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2528. 2024-11-12T14:19:41,671 DEBUG [Thread-2528 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-12T14:19:41,671 DEBUG [Thread-2528 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:41,671 DEBUG [Thread-2528 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:41,671 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:19:41,671 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-12T14:19:41,671 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 32 2024-11-12T14:19:41,671 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 108 2024-11-12T14:19:41,671 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 43 2024-11-12T14:19:41,671 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-12T14:19:41,671 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-11-12T14:19:41,671 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-12T14:19:41,671 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2638 2024-11-12T14:19:41,671 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2588 2024-11-12T14:19:41,671 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2544 2024-11-12T14:19:41,671 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2629 2024-11-12T14:19:41,671 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2584 2024-11-12T14:19:41,671 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-12T14:19:41,671 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T14:19:41,671 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T14:19:41,671 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:41,672 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:41,672 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:19:41,672 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-12T14:19:41,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-12T14:19:41,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T14:19:41,676 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731421181676"}]},"ts":"1731421181676"} 2024-11-12T14:19:41,677 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-12T14:19:41,677 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-12T14:19:41,678 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-12T14:19:41,680 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1aa16d140ed3f62a95c849e7bf6cc7a, UNASSIGN}] 2024-11-12T14:19:41,681 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1aa16d140ed3f62a95c849e7bf6cc7a, UNASSIGN 2024-11-12T14:19:41,681 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=e1aa16d140ed3f62a95c849e7bf6cc7a, regionState=CLOSING, regionLocation=2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:41,682 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=164, ppid=163, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1aa16d140ed3f62a95c849e7bf6cc7a, UNASSIGN because future has completed 2024-11-12T14:19:41,683 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-12T14:19:41,683 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822}] 2024-11-12T14:19:41,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T14:19:41,835 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(122): Close e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:41,835 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-12T14:19:41,835 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1722): Closing e1aa16d140ed3f62a95c849e7bf6cc7a, disabling compactions & flushes 2024-11-12T14:19:41,835 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:41,835 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:41,835 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. after waiting 0 ms 2024-11-12T14:19:41,835 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:41,835 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(2902): Flushing e1aa16d140ed3f62a95c849e7bf6cc7a 3/3 column families, dataSize=26.84 KB heapSize=80.44 KB 2024-11-12T14:19:41,835 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=A 2024-11-12T14:19:41,835 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:41,835 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=B 2024-11-12T14:19:41,835 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:41,835 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactingMemStore(205): FLUSHING TO DISK e1aa16d140ed3f62a95c849e7bf6cc7a, store=C 2024-11-12T14:19:41,835 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.CompactionPipeline(136): Swapping pipeline suffix; before=1, new segment=null 2024-11-12T14:19:41,839 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ee1f930b4f0d4ea48c17c8023a78600c_e1aa16d140ed3f62a95c849e7bf6cc7a is 50, key is test_row_0/A:col10/1731421179160/Put/seqid=0 2024-11-12T14:19:41,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742520_1696 (size=9914) 2024-11-12T14:19:41,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T14:19:42,242 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-12T14:19:42,246 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241112ee1f930b4f0d4ea48c17c8023a78600c_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ee1f930b4f0d4ea48c17c8023a78600c_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:42,246 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/74bf12651fa14f508e3d41b00de71a35, store: [table=TestAcidGuarantees family=A region=e1aa16d140ed3f62a95c849e7bf6cc7a] 2024-11-12T14:19:42,247 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/74bf12651fa14f508e3d41b00de71a35 is 175, key is test_row_0/A:col10/1731421179160/Put/seqid=0 2024-11-12T14:19:42,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742521_1697 (size=22561) 2024-11-12T14:19:42,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T14:19:42,651 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=426, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/74bf12651fa14f508e3d41b00de71a35 2024-11-12T14:19:42,655 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/af3d252d2c43498b8d3c85db935d72eb is 50, key is test_row_0/B:col10/1731421179160/Put/seqid=0 2024-11-12T14:19:42,658 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742522_1698 (size=9857) 2024-11-12T14:19:42,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T14:19:43,059 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/af3d252d2c43498b8d3c85db935d72eb 2024-11-12T14:19:43,064 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/35ad485219844ed78113cf14bf44451d is 50, key is test_row_0/C:col10/1731421179160/Put/seqid=0 2024-11-12T14:19:43,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742523_1699 (size=9857) 2024-11-12T14:19:43,467 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/35ad485219844ed78113cf14bf44451d 2024-11-12T14:19:43,470 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/A/74bf12651fa14f508e3d41b00de71a35 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/74bf12651fa14f508e3d41b00de71a35 2024-11-12T14:19:43,473 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/74bf12651fa14f508e3d41b00de71a35, entries=100, sequenceid=426, filesize=22.0 K 2024-11-12T14:19:43,473 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/B/af3d252d2c43498b8d3c85db935d72eb as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/af3d252d2c43498b8d3c85db935d72eb 2024-11-12T14:19:43,475 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/af3d252d2c43498b8d3c85db935d72eb, entries=100, sequenceid=426, filesize=9.6 K 2024-11-12T14:19:43,476 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/.tmp/C/35ad485219844ed78113cf14bf44451d as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/35ad485219844ed78113cf14bf44451d 2024-11-12T14:19:43,478 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/35ad485219844ed78113cf14bf44451d, entries=100, sequenceid=426, filesize=9.6 K 2024-11-12T14:19:43,479 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for e1aa16d140ed3f62a95c849e7bf6cc7a in 1643ms, sequenceid=426, compaction requested=false 2024-11-12T14:19:43,479 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/8455b70638034900aac82439d633619f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/04e2e9800ac44fa880d2e689cbdec5d5, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/be3c126d3bca491d86acc896c21fb04a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/a806108ad99544e084bcb3278bbba054, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/2244a0ffd17044fc91a42ec28d73a96f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/85547f1332d4440981510c878887b607, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/f5949ebfcbee4e5687266bf370b5239e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/cba804fdea45429ab0b8658d33065559, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/b7366b3a299e447289a91cc9afd3c857, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/23c3e751e6af42b3bce1c56724d58f51, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/552c88da6b4c47b480e1603b3268ca45, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/f6657545e8d64d4eb2d61e846329fbc4, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/a5446c66869d4f9588d19f2cf748b24f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/b007d9ed20ad4a169b68488e2b7a8452, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/2d6bef64705644ac842b5c9ad57af5ca, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/4ce938e760a04a3baca9b480ea5f52f0, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/69b761b4ab84467387b52f7b2a75f56b, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/454275c5db4b440eae27417a788332c5, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/e40542cdd9aa45e8b5b52fb789ff630c, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/cb21b690e92b4100925a2eec83c0dd3a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/96ea226a39c4427fa9c8b164313961f4, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/1b0438a977114e7daeeb59c085728238, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/3bd83b4f04d3430eba7bad04705e414b, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/c7b1cefc507544938dcf3980422318ec, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/5cc9b7d4e41b42dda7f3c05a7ba3142a, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/9d9873b1afd54278af246f3fdd0b3414, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/df37ba2636db46c88ede0c8dd285dc39, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/4e5cfac9ad2644058c412693800fb444, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/8b1d9f325e694da1a3ed381d49bc417b, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/3b09656d168d4164ba73dd630b011f43, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/14978444336d4067a642ce314407ed14] to archive 2024-11-12T14:19:43,480 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T14:19:43,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/8455b70638034900aac82439d633619f to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/8455b70638034900aac82439d633619f 2024-11-12T14:19:43,481 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/04e2e9800ac44fa880d2e689cbdec5d5 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/04e2e9800ac44fa880d2e689cbdec5d5 2024-11-12T14:19:43,482 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/be3c126d3bca491d86acc896c21fb04a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/be3c126d3bca491d86acc896c21fb04a 2024-11-12T14:19:43,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/a806108ad99544e084bcb3278bbba054 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/a806108ad99544e084bcb3278bbba054 2024-11-12T14:19:43,483 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/2244a0ffd17044fc91a42ec28d73a96f to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/2244a0ffd17044fc91a42ec28d73a96f 2024-11-12T14:19:43,484 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/85547f1332d4440981510c878887b607 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/85547f1332d4440981510c878887b607 2024-11-12T14:19:43,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/f5949ebfcbee4e5687266bf370b5239e to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/f5949ebfcbee4e5687266bf370b5239e 2024-11-12T14:19:43,485 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/cba804fdea45429ab0b8658d33065559 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/cba804fdea45429ab0b8658d33065559 2024-11-12T14:19:43,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/b7366b3a299e447289a91cc9afd3c857 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/b7366b3a299e447289a91cc9afd3c857 2024-11-12T14:19:43,486 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/23c3e751e6af42b3bce1c56724d58f51 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/23c3e751e6af42b3bce1c56724d58f51 2024-11-12T14:19:43,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/552c88da6b4c47b480e1603b3268ca45 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/552c88da6b4c47b480e1603b3268ca45 2024-11-12T14:19:43,487 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/f6657545e8d64d4eb2d61e846329fbc4 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/f6657545e8d64d4eb2d61e846329fbc4 2024-11-12T14:19:43,488 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/a5446c66869d4f9588d19f2cf748b24f to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/a5446c66869d4f9588d19f2cf748b24f 2024-11-12T14:19:43,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/b007d9ed20ad4a169b68488e2b7a8452 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/b007d9ed20ad4a169b68488e2b7a8452 2024-11-12T14:19:43,489 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/2d6bef64705644ac842b5c9ad57af5ca to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/2d6bef64705644ac842b5c9ad57af5ca 2024-11-12T14:19:43,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/4ce938e760a04a3baca9b480ea5f52f0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/4ce938e760a04a3baca9b480ea5f52f0 2024-11-12T14:19:43,490 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/69b761b4ab84467387b52f7b2a75f56b to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/69b761b4ab84467387b52f7b2a75f56b 2024-11-12T14:19:43,491 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/454275c5db4b440eae27417a788332c5 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/454275c5db4b440eae27417a788332c5 2024-11-12T14:19:43,492 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/e40542cdd9aa45e8b5b52fb789ff630c to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/e40542cdd9aa45e8b5b52fb789ff630c 2024-11-12T14:19:43,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/cb21b690e92b4100925a2eec83c0dd3a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/cb21b690e92b4100925a2eec83c0dd3a 2024-11-12T14:19:43,493 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/96ea226a39c4427fa9c8b164313961f4 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/96ea226a39c4427fa9c8b164313961f4 2024-11-12T14:19:43,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/1b0438a977114e7daeeb59c085728238 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/1b0438a977114e7daeeb59c085728238 2024-11-12T14:19:43,495 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/3bd83b4f04d3430eba7bad04705e414b to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/3bd83b4f04d3430eba7bad04705e414b 2024-11-12T14:19:43,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/c7b1cefc507544938dcf3980422318ec to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/c7b1cefc507544938dcf3980422318ec 2024-11-12T14:19:43,496 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/5cc9b7d4e41b42dda7f3c05a7ba3142a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/5cc9b7d4e41b42dda7f3c05a7ba3142a 2024-11-12T14:19:43,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/9d9873b1afd54278af246f3fdd0b3414 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/9d9873b1afd54278af246f3fdd0b3414 2024-11-12T14:19:43,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/df37ba2636db46c88ede0c8dd285dc39 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/df37ba2636db46c88ede0c8dd285dc39 2024-11-12T14:19:43,498 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/4e5cfac9ad2644058c412693800fb444 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/4e5cfac9ad2644058c412693800fb444 2024-11-12T14:19:43,499 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/8b1d9f325e694da1a3ed381d49bc417b to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/8b1d9f325e694da1a3ed381d49bc417b 2024-11-12T14:19:43,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/3b09656d168d4164ba73dd630b011f43 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/3b09656d168d4164ba73dd630b011f43 2024-11-12T14:19:43,500 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/14978444336d4067a642ce314407ed14 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/14978444336d4067a642ce314407ed14 2024-11-12T14:19:43,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/35ca1eddfda44f5c94a6e4ecd140ca67, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/640e88ef02ed4aa5a52fac019048b96d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/c292014e9b274a73a1df7034b707c602, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/4836afae9c0c47209581d5af758eaa70, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/ea6774aa8886452f82ac99a16c962744, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/d59297916ed24b6ea3f63c2f014a9455, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/31ad260bda2343af9f6dab8cca4a4299, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/09385d7a5b20481cb75c65e4271772d5, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/1c1f93d72d7641ef806a29895fcdeb75, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/5ffaecee499b484d91f5b61c8fc27e02, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/8c68977682074f54adc6a38e7425b24f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/cc6210e6e97943af9ced36691c6f925d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/7c8c639943b14904ae5741a87f187db5, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/d0ceee2501ba4e518b68b55238db04fd, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/bf9635e8dbd44c77afccb69e08035c3e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/92ed1228396247bf894d2c3929ce14e3, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/96365ebadcf046978831bbf67cccd0ed, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/7560f58b05ac4ab599f3939240c84f90, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/ab5cbd9d6da14b5c99285c65f36bd5d4, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/406560d39dc44b2da671903b3f338eb7, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/b2d47860dbed4543be0b1f203407546f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/133cb7010c6045da85da80a1f838e424, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/8bd0cbee31fb4f52a5a3f8982d63babc, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/3f890509eda9499887e09a402178e2b9, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/936c949bd4464f37955092e6715a4c65, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/8618b20e54d64b9c80fd299e569f0f6d, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/208794e3a2704603808fae4c39776f88, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/32c6a9a278dd48d5bd7ce1b9f8e7bd5e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/c33c5de9ebce4ae0849c7bff5c49a422, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/1bc626d526de461dbf0184ba6c2159ec, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/b780e2478f3e4f46ac9f6777c0b0370c] to archive 2024-11-12T14:19:43,502 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T14:19:43,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/35ca1eddfda44f5c94a6e4ecd140ca67 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/35ca1eddfda44f5c94a6e4ecd140ca67 2024-11-12T14:19:43,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/640e88ef02ed4aa5a52fac019048b96d to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/640e88ef02ed4aa5a52fac019048b96d 2024-11-12T14:19:43,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/c292014e9b274a73a1df7034b707c602 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/c292014e9b274a73a1df7034b707c602 2024-11-12T14:19:43,505 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/4836afae9c0c47209581d5af758eaa70 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/4836afae9c0c47209581d5af758eaa70 2024-11-12T14:19:43,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/ea6774aa8886452f82ac99a16c962744 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/ea6774aa8886452f82ac99a16c962744 2024-11-12T14:19:43,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/d59297916ed24b6ea3f63c2f014a9455 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/d59297916ed24b6ea3f63c2f014a9455 2024-11-12T14:19:43,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/31ad260bda2343af9f6dab8cca4a4299 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/31ad260bda2343af9f6dab8cca4a4299 2024-11-12T14:19:43,508 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/09385d7a5b20481cb75c65e4271772d5 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/09385d7a5b20481cb75c65e4271772d5 2024-11-12T14:19:43,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/1c1f93d72d7641ef806a29895fcdeb75 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/1c1f93d72d7641ef806a29895fcdeb75 2024-11-12T14:19:43,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/5ffaecee499b484d91f5b61c8fc27e02 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/5ffaecee499b484d91f5b61c8fc27e02 2024-11-12T14:19:43,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/8c68977682074f54adc6a38e7425b24f to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/8c68977682074f54adc6a38e7425b24f 2024-11-12T14:19:43,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/cc6210e6e97943af9ced36691c6f925d to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/cc6210e6e97943af9ced36691c6f925d 2024-11-12T14:19:43,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/7c8c639943b14904ae5741a87f187db5 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/7c8c639943b14904ae5741a87f187db5 2024-11-12T14:19:43,512 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/d0ceee2501ba4e518b68b55238db04fd to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/d0ceee2501ba4e518b68b55238db04fd 2024-11-12T14:19:43,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/bf9635e8dbd44c77afccb69e08035c3e to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/bf9635e8dbd44c77afccb69e08035c3e 2024-11-12T14:19:43,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/92ed1228396247bf894d2c3929ce14e3 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/92ed1228396247bf894d2c3929ce14e3 2024-11-12T14:19:43,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/96365ebadcf046978831bbf67cccd0ed to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/96365ebadcf046978831bbf67cccd0ed 2024-11-12T14:19:43,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/7560f58b05ac4ab599f3939240c84f90 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/7560f58b05ac4ab599f3939240c84f90 2024-11-12T14:19:43,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/ab5cbd9d6da14b5c99285c65f36bd5d4 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/ab5cbd9d6da14b5c99285c65f36bd5d4 2024-11-12T14:19:43,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/406560d39dc44b2da671903b3f338eb7 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/406560d39dc44b2da671903b3f338eb7 2024-11-12T14:19:43,517 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/b2d47860dbed4543be0b1f203407546f to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/b2d47860dbed4543be0b1f203407546f 2024-11-12T14:19:43,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/133cb7010c6045da85da80a1f838e424 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/133cb7010c6045da85da80a1f838e424 2024-11-12T14:19:43,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/8bd0cbee31fb4f52a5a3f8982d63babc to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/8bd0cbee31fb4f52a5a3f8982d63babc 2024-11-12T14:19:43,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/3f890509eda9499887e09a402178e2b9 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/3f890509eda9499887e09a402178e2b9 2024-11-12T14:19:43,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/936c949bd4464f37955092e6715a4c65 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/936c949bd4464f37955092e6715a4c65 2024-11-12T14:19:43,521 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/8618b20e54d64b9c80fd299e569f0f6d to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/8618b20e54d64b9c80fd299e569f0f6d 2024-11-12T14:19:43,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/208794e3a2704603808fae4c39776f88 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/208794e3a2704603808fae4c39776f88 2024-11-12T14:19:43,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/32c6a9a278dd48d5bd7ce1b9f8e7bd5e to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/32c6a9a278dd48d5bd7ce1b9f8e7bd5e 2024-11-12T14:19:43,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/c33c5de9ebce4ae0849c7bff5c49a422 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/c33c5de9ebce4ae0849c7bff5c49a422 2024-11-12T14:19:43,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/1bc626d526de461dbf0184ba6c2159ec to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/1bc626d526de461dbf0184ba6c2159ec 2024-11-12T14:19:43,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/b780e2478f3e4f46ac9f6777c0b0370c to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/b780e2478f3e4f46ac9f6777c0b0370c 2024-11-12T14:19:43,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/17b12a50b8294ab8b7339fd5fa6eb5f1, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/7cd769140fb34b94aef8b4fca7e6c312, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/eec7522f0081416c99830e33d53f9f2e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/916993ff540142148b84028c608eeec9, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/7f09ee89d76542cda81751a4dd12d91e, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/03f2be16f5f843929d8b25d276f94d88, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/ed22870873c143aa93a09ce000ee5c85, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/7a7a04b7327f4ea880f837a30e43e33b, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/70e52341f25a4a6ca8cb62e10fb9e994, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/2f3da4e666bd4680a2b4266df8ceb003, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/c8f609d8605e43f6b11f6ba4844d7586, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/c6a2bb18b5584c0c8c4051d57bb3bb92, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/299802d71c914b879d06b2e97b8bfbc1, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/8373c2c6b3514c2f899dd23aab7c3937, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/c4bfd34593634b909c19fdf35ca9c562, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/2efbf91ce4e34786b7244800674cf969, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/35df06708ba6438b91692f9325cea0e6, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/3c892856a58241a98d054b6ae0857b87, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/763fc6f7b5f04c6491b82454b048bb66, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/db9f2783f99445458c5fbeed4e25f3e0, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/4753b75e3183448c8344955753ee8669, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/54c68c3feffc45b4aea3a70462851016, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/3c66a12905d44fd98c7be3c76e0448d0, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/ca8cd0ce39cd4b6db0a4c6cfbb758cae, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/da0ed1b128134abe8ef7ebdf05e71774, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/261129df156b4f52883f1ef99ef23a4f, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/fa9b9f0e454648639f75e18107ad6676, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/b080fa71f7de42d08bc81ed3f346e608, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/ab94fc0d69134181a77d5c148a844655, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/b162878efd4b411c9223a3b06dc3cce3, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/c17c39008f114ecca5a8fb9d75064bed] to archive 2024-11-12T14:19:43,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-12T14:19:43,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/17b12a50b8294ab8b7339fd5fa6eb5f1 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/17b12a50b8294ab8b7339fd5fa6eb5f1 2024-11-12T14:19:43,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/7cd769140fb34b94aef8b4fca7e6c312 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/7cd769140fb34b94aef8b4fca7e6c312 2024-11-12T14:19:43,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/eec7522f0081416c99830e33d53f9f2e to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/eec7522f0081416c99830e33d53f9f2e 2024-11-12T14:19:43,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/916993ff540142148b84028c608eeec9 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/916993ff540142148b84028c608eeec9 2024-11-12T14:19:43,531 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/7f09ee89d76542cda81751a4dd12d91e to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/7f09ee89d76542cda81751a4dd12d91e 2024-11-12T14:19:43,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/03f2be16f5f843929d8b25d276f94d88 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/03f2be16f5f843929d8b25d276f94d88 2024-11-12T14:19:43,533 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/ed22870873c143aa93a09ce000ee5c85 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/ed22870873c143aa93a09ce000ee5c85 2024-11-12T14:19:43,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/7a7a04b7327f4ea880f837a30e43e33b to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/7a7a04b7327f4ea880f837a30e43e33b 2024-11-12T14:19:43,534 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/70e52341f25a4a6ca8cb62e10fb9e994 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/70e52341f25a4a6ca8cb62e10fb9e994 2024-11-12T14:19:43,535 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/2f3da4e666bd4680a2b4266df8ceb003 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/2f3da4e666bd4680a2b4266df8ceb003 2024-11-12T14:19:43,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/c8f609d8605e43f6b11f6ba4844d7586 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/c8f609d8605e43f6b11f6ba4844d7586 2024-11-12T14:19:43,536 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/c6a2bb18b5584c0c8c4051d57bb3bb92 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/c6a2bb18b5584c0c8c4051d57bb3bb92 2024-11-12T14:19:43,537 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/299802d71c914b879d06b2e97b8bfbc1 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/299802d71c914b879d06b2e97b8bfbc1 2024-11-12T14:19:43,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/8373c2c6b3514c2f899dd23aab7c3937 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/8373c2c6b3514c2f899dd23aab7c3937 2024-11-12T14:19:43,538 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/c4bfd34593634b909c19fdf35ca9c562 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/c4bfd34593634b909c19fdf35ca9c562 2024-11-12T14:19:43,539 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/2efbf91ce4e34786b7244800674cf969 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/2efbf91ce4e34786b7244800674cf969 2024-11-12T14:19:43,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/35df06708ba6438b91692f9325cea0e6 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/35df06708ba6438b91692f9325cea0e6 2024-11-12T14:19:43,540 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/3c892856a58241a98d054b6ae0857b87 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/3c892856a58241a98d054b6ae0857b87 2024-11-12T14:19:43,541 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/763fc6f7b5f04c6491b82454b048bb66 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/763fc6f7b5f04c6491b82454b048bb66 2024-11-12T14:19:43,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/db9f2783f99445458c5fbeed4e25f3e0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/db9f2783f99445458c5fbeed4e25f3e0 2024-11-12T14:19:43,542 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/4753b75e3183448c8344955753ee8669 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/4753b75e3183448c8344955753ee8669 2024-11-12T14:19:43,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/54c68c3feffc45b4aea3a70462851016 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/54c68c3feffc45b4aea3a70462851016 2024-11-12T14:19:43,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/3c66a12905d44fd98c7be3c76e0448d0 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/3c66a12905d44fd98c7be3c76e0448d0 2024-11-12T14:19:43,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/ca8cd0ce39cd4b6db0a4c6cfbb758cae to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/ca8cd0ce39cd4b6db0a4c6cfbb758cae 2024-11-12T14:19:43,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/da0ed1b128134abe8ef7ebdf05e71774 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/da0ed1b128134abe8ef7ebdf05e71774 2024-11-12T14:19:43,545 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/261129df156b4f52883f1ef99ef23a4f to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/261129df156b4f52883f1ef99ef23a4f 2024-11-12T14:19:43,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/fa9b9f0e454648639f75e18107ad6676 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/fa9b9f0e454648639f75e18107ad6676 2024-11-12T14:19:43,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/b080fa71f7de42d08bc81ed3f346e608 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/b080fa71f7de42d08bc81ed3f346e608 2024-11-12T14:19:43,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/ab94fc0d69134181a77d5c148a844655 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/ab94fc0d69134181a77d5c148a844655 2024-11-12T14:19:43,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/b162878efd4b411c9223a3b06dc3cce3 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/b162878efd4b411c9223a3b06dc3cce3 2024-11-12T14:19:43,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/c17c39008f114ecca5a8fb9d75064bed to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/c17c39008f114ecca5a8fb9d75064bed 2024-11-12T14:19:43,552 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/recovered.edits/429.seqid, newMaxSeqId=429, maxSeqId=4 2024-11-12T14:19:43,552 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. 2024-11-12T14:19:43,552 DEBUG [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] regionserver.HRegion(1676): Region close journal for e1aa16d140ed3f62a95c849e7bf6cc7a: Waiting for close lock at 1731421181835Running coprocessor pre-close hooks at 1731421181835Disabling compacts and flushes for region at 1731421181835Disabling writes for close at 1731421181835Obtaining lock to block concurrent updates at 1731421181835Preparing flush snapshotting stores in e1aa16d140ed3f62a95c849e7bf6cc7a at 1731421181835Finished memstore snapshotting TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=82320, getOffHeapSize=0, getCellsCount=600 at 1731421181835Flushing stores of TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a. at 1731421181836 (+1 ms)Flushing e1aa16d140ed3f62a95c849e7bf6cc7a/A: creating writer at 1731421181836Flushing mob file e1aa16d140ed3f62a95c849e7bf6cc7a/A: appending metadata at 1731421181839 (+3 ms)Flushing mob file e1aa16d140ed3f62a95c849e7bf6cc7a/A: closing flushed file at 1731421181839Flushing e1aa16d140ed3f62a95c849e7bf6cc7a/A: appending metadata at 1731421182246 (+407 ms)Flushing e1aa16d140ed3f62a95c849e7bf6cc7a/A: closing flushed file at 1731421182246Flushing e1aa16d140ed3f62a95c849e7bf6cc7a/B: creating writer at 1731421182653 (+407 ms)Flushing e1aa16d140ed3f62a95c849e7bf6cc7a/B: appending metadata at 1731421182655 (+2 ms)Flushing e1aa16d140ed3f62a95c849e7bf6cc7a/B: closing flushed file at 1731421182655Flushing e1aa16d140ed3f62a95c849e7bf6cc7a/C: creating writer at 1731421183062 (+407 ms)Flushing e1aa16d140ed3f62a95c849e7bf6cc7a/C: appending metadata at 1731421183063 (+1 ms)Flushing e1aa16d140ed3f62a95c849e7bf6cc7a/C: closing flushed file at 1731421183063Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6ce572c9: reopening flushed file at 1731421183470 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@107b2478: reopening flushed file at 1731421183473 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c5ef292: reopening flushed file at 1731421183475 (+2 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~80.39 KB/82320, currentSize=0 B/0 for e1aa16d140ed3f62a95c849e7bf6cc7a in 1643ms, sequenceid=426, compaction requested=false at 1731421183479 (+4 ms)Writing region close event to WAL at 1731421183550 (+71 ms)Running coprocessor post-close hooks at 1731421183552 (+2 ms)Closed at 1731421183552 2024-11-12T14:19:43,554 INFO [RS_CLOSE_REGION-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_REGION, pid=165}] handler.UnassignRegionHandler(157): Closed e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:43,554 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=164 updating hbase:meta row=e1aa16d140ed3f62a95c849e7bf6cc7a, regionState=CLOSED 2024-11-12T14:19:43,555 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=165, ppid=164, state=RUNNABLE, hasLock=false; CloseRegionProcedure e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 because future has completed 2024-11-12T14:19:43,557 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-12T14:19:43,557 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseRegionProcedure e1aa16d140ed3f62a95c849e7bf6cc7a, server=2b6d221c5cde,42925,1731420990822 in 1.8730 sec 2024-11-12T14:19:43,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=164, resume processing ppid=163 2024-11-12T14:19:43,558 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, ppid=163, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=e1aa16d140ed3f62a95c849e7bf6cc7a, UNASSIGN in 1.8770 sec 2024-11-12T14:19:43,560 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-12T14:19:43,560 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8810 sec 2024-11-12T14:19:43,561 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731421183561"}]},"ts":"1731421183561"} 2024-11-12T14:19:43,563 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-12T14:19:43,563 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-12T14:19:43,564 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.8910 sec 2024-11-12T14:19:43,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-12T14:19:43,820 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:19:43,820 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-12T14:19:43,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:19:43,822 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=166, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:19:43,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-12T14:19:43,822 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=166, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:19:43,824 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:43,826 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A, FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B, FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C, FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/recovered.edits] 2024-11-12T14:19:43,828 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/1647fcd9ed584ed68616622fcac81417 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/1647fcd9ed584ed68616622fcac81417 2024-11-12T14:19:43,829 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/74bf12651fa14f508e3d41b00de71a35 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/A/74bf12651fa14f508e3d41b00de71a35 2024-11-12T14:19:43,830 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/5db46b1e955b4fbeb9bf6667278a5825 to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/5db46b1e955b4fbeb9bf6667278a5825 2024-11-12T14:19:43,830 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/af3d252d2c43498b8d3c85db935d72eb to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/B/af3d252d2c43498b8d3c85db935d72eb 2024-11-12T14:19:43,832 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/35ad485219844ed78113cf14bf44451d to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/35ad485219844ed78113cf14bf44451d 2024-11-12T14:19:43,832 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/b2a175111577431da9248f60355012aa to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/C/b2a175111577431da9248f60355012aa 2024-11-12T14:19:43,834 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/recovered.edits/429.seqid to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a/recovered.edits/429.seqid 2024-11-12T14:19:43,834 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/default/TestAcidGuarantees/e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:43,834 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-12T14:19:43,834 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T14:19:43,835 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-12T14:19:43,837 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112035812fa48264f7387f0e04b32e659e0_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112035812fa48264f7387f0e04b32e659e0_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:43,837 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111210e2d5edf8d44eaba84a43e103bf45e7_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111210e2d5edf8d44eaba84a43e103bf45e7_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:43,838 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123491b1b36ac44789af5bea616ff1d24e_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123491b1b36ac44789af5bea616ff1d24e_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:43,839 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123dd83a604cc248cd9febf0539327ad20_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411123dd83a604cc248cd9febf0539327ad20_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:43,839 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111241ba9ebab9604b2faea227409f2fe620_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111241ba9ebab9604b2faea227409f2fe620_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:43,840 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124cd01ccc6f554cdb8d4d29afcb5a2bc4_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411124cd01ccc6f554cdb8d4d29afcb5a2bc4_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:43,841 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125422d87f509f47e5b5e6a35f03746ca5_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125422d87f509f47e5b5e6a35f03746ca5_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:43,842 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111257f432a823ba47099132d06833221eaa_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111257f432a823ba47099132d06833221eaa_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:43,843 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125cfa2851b5784cc995c3dee400a32604_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411125cfa2851b5784cc995c3dee400a32604_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:43,843 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111279a1cf6eb0bc4078ae7e3d9c74ac88db_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111279a1cf6eb0bc4078ae7e3d9c74ac88db_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:43,844 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112909780ea756b4440af5cdaa8f393bdde_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112909780ea756b4440af5cdaa8f393bdde_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:43,845 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111298bfcd08b1e0465289c02fddf5b8fd92_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111298bfcd08b1e0465289c02fddf5b8fd92_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:43,845 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129d4673713f6747c38cadf67452805a21_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129d4673713f6747c38cadf67452805a21_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:43,846 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129ee4dcd58778494ca43dba94b64f81e1_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411129ee4dcd58778494ca43dba94b64f81e1_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:43,847 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b1484f2ad20e49e9b3796f0fee2839f5_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b1484f2ad20e49e9b3796f0fee2839f5_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:43,847 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b943b5a6c4504e588069d107ad2d7acd_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112b943b5a6c4504e588069d107ad2d7acd_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:43,848 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112be83a2823a8b4e1d807f1ba71011c4d8_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112be83a2823a8b4e1d807f1ba71011c4d8_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:43,849 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c0317c9b8d7447efa54a7a517dfe40c2_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c0317c9b8d7447efa54a7a517dfe40c2_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:43,849 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c20f5ec30ea54a56a034b880814142e6_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c20f5ec30ea54a56a034b880814142e6_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:43,850 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c41b0842b27c42dca2cf1f1f200f4622_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112c41b0842b27c42dca2cf1f1f200f4622_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:43,851 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d2f8d20a2c6d471b86c1f9215e79ba9f_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112d2f8d20a2c6d471b86c1f9215e79ba9f_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:43,851 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e57639a7b7cc4ba48221e282b5b13acf_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e57639a7b7cc4ba48221e282b5b13acf_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:43,852 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e701ae80a27646cba31229d48e19bea3_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112e701ae80a27646cba31229d48e19bea3_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:43,852 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ee1f930b4f0d4ea48c17c8023a78600c_e1aa16d140ed3f62a95c849e7bf6cc7a to hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241112ee1f930b4f0d4ea48c17c8023a78600c_e1aa16d140ed3f62a95c849e7bf6cc7a 2024-11-12T14:19:43,853 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-12T14:19:43,854 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=166, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:19:43,856 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-12T14:19:43,858 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-12T14:19:43,859 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=166, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:19:43,859 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-12T14:19:43,859 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731421183859"}]},"ts":"9223372036854775807"} 2024-11-12T14:19:43,861 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-12T14:19:43,861 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => e1aa16d140ed3f62a95c849e7bf6cc7a, NAME => 'TestAcidGuarantees,,1731421150173.e1aa16d140ed3f62a95c849e7bf6cc7a.', STARTKEY => '', ENDKEY => ''}] 2024-11-12T14:19:43,861 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-12T14:19:43,861 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731421183861"}]},"ts":"9223372036854775807"} 2024-11-12T14:19:43,863 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-12T14:19:43,863 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=166, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-12T14:19:43,864 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 43 msec 2024-11-12T14:19:43,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45591 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-12T14:19:43,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-12T14:19:43,930 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-12T14:19:43,937 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithEagerPolicy#testMobGetAtomicity Thread=248 (was 246) - Thread LEAK? -, OpenFileDescriptor=447 (was 449), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=728 (was 833), ProcessCount=11 (was 11), AvailableMemoryMB=861 (was 397) - AvailableMemoryMB LEAK? - 2024-11-12T14:19:43,937 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-12T14:19:43,937 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T14:19:43,937 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T14:19:43,937 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:43,937 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:43,937 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-12T14:19:43,937 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-12T14:19:43,937 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1336910459, stopped=false 2024-11-12T14:19:43,938 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=2b6d221c5cde,45591,1731420989939 2024-11-12T14:19:43,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42925-0x1012f67a1e60001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T14:19:43,950 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45591-0x1012f67a1e60000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-12T14:19:43,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42925-0x1012f67a1e60001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:19:43,951 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45591-0x1012f67a1e60000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:19:43,951 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T14:19:43,951 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-12T14:19:43,951 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T14:19:43,951 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:43,951 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '2b6d221c5cde,42925,1731420990822' ***** 2024-11-12T14:19:43,951 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-12T14:19:43,951 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:45591-0x1012f67a1e60000, quorum=127.0.0.1:50547, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T14:19:43,951 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42925-0x1012f67a1e60001, quorum=127.0.0.1:50547, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-12T14:19:43,952 INFO [RS:0;2b6d221c5cde:42925 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-12T14:19:43,952 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-12T14:19:43,952 INFO [RS:0;2b6d221c5cde:42925 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-12T14:19:43,952 INFO [RS:0;2b6d221c5cde:42925 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-12T14:19:43,952 INFO [RS:0;2b6d221c5cde:42925 {}] regionserver.HRegionServer(959): stopping server 2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:43,952 INFO [RS:0;2b6d221c5cde:42925 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T14:19:43,952 INFO [RS:0;2b6d221c5cde:42925 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;2b6d221c5cde:42925. 2024-11-12T14:19:43,952 DEBUG [RS:0;2b6d221c5cde:42925 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-12T14:19:43,952 DEBUG [RS:0;2b6d221c5cde:42925 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:43,953 INFO [RS:0;2b6d221c5cde:42925 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-12T14:19:43,953 INFO [RS:0;2b6d221c5cde:42925 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-12T14:19:43,953 INFO [RS:0;2b6d221c5cde:42925 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-12T14:19:43,953 INFO [RS:0;2b6d221c5cde:42925 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-12T14:19:43,953 INFO [RS:0;2b6d221c5cde:42925 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-12T14:19:43,953 DEBUG [RS:0;2b6d221c5cde:42925 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-12T14:19:43,953 DEBUG [RS:0;2b6d221c5cde:42925 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T14:19:43,953 DEBUG [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-12T14:19:43,954 INFO [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-12T14:19:43,954 DEBUG [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-12T14:19:43,954 DEBUG [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-12T14:19:43,954 DEBUG [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-12T14:19:43,954 INFO [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-12T14:19:43,958 INFO [regionserver/2b6d221c5cde:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-12T14:19:43,958 INFO [regionserver/2b6d221c5cde:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-12T14:19:43,974 DEBUG [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/hbase/meta/1588230740/.tmp/info/18aac9a3a46a4b4ea76c3a142c50d211 is 95, key is TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254./info:/1731421023214/DeleteFamily/seqid=0 2024-11-12T14:19:43,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742524_1700 (size=6514) 2024-11-12T14:19:43,984 INFO [regionserver/2b6d221c5cde:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T14:19:44,154 DEBUG [RS:0;2b6d221c5cde:42925 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T14:19:44,354 DEBUG [RS:0;2b6d221c5cde:42925 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T14:19:44,377 INFO [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/hbase/meta/1588230740/.tmp/info/18aac9a3a46a4b4ea76c3a142c50d211 2024-11-12T14:19:44,393 DEBUG [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/hbase/meta/1588230740/.tmp/ns/76c9eed7a2024110a3e9c933d89a49de is 93, key is TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254./ns:/1731421023153/DeleteFamily/seqid=0 2024-11-12T14:19:44,396 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742525_1701 (size=5906) 2024-11-12T14:19:44,554 DEBUG [RS:0;2b6d221c5cde:42925 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T14:19:44,754 DEBUG [RS:0;2b6d221c5cde:42925 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T14:19:44,796 INFO [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/hbase/meta/1588230740/.tmp/ns/76c9eed7a2024110a3e9c933d89a49de 2024-11-12T14:19:44,813 DEBUG [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/hbase/meta/1588230740/.tmp/rep_barrier/0ba964ec5e5942c4bbb0252a3012363f is 102, key is TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254./rep_barrier:/1731421023153/DeleteFamily/seqid=0 2024-11-12T14:19:44,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742526_1702 (size=6025) 2024-11-12T14:19:44,954 DEBUG [RS:0;2b6d221c5cde:42925 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T14:19:45,155 DEBUG [RS:0;2b6d221c5cde:42925 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T14:19:45,216 INFO [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/hbase/meta/1588230740/.tmp/rep_barrier/0ba964ec5e5942c4bbb0252a3012363f 2024-11-12T14:19:45,232 DEBUG [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/hbase/meta/1588230740/.tmp/table/d5d35076ec0e45feb567504c25b6ef1b is 96, key is TestAcidGuarantees,,1731420996415.6ea61e7c1d124f78e4a5241afdaab254./table:/1731421023153/DeleteFamily/seqid=0 2024-11-12T14:19:45,235 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742527_1703 (size=5936) 2024-11-12T14:19:45,355 DEBUG [RS:0;2b6d221c5cde:42925 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T14:19:45,555 DEBUG [RS:0;2b6d221c5cde:42925 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-12T14:19:45,635 INFO [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/hbase/meta/1588230740/.tmp/table/d5d35076ec0e45feb567504c25b6ef1b 2024-11-12T14:19:45,639 DEBUG [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/hbase/meta/1588230740/.tmp/info/18aac9a3a46a4b4ea76c3a142c50d211 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/hbase/meta/1588230740/info/18aac9a3a46a4b4ea76c3a142c50d211 2024-11-12T14:19:45,641 INFO [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/hbase/meta/1588230740/info/18aac9a3a46a4b4ea76c3a142c50d211, entries=12, sequenceid=90, filesize=6.4 K 2024-11-12T14:19:45,642 DEBUG [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/hbase/meta/1588230740/.tmp/ns/76c9eed7a2024110a3e9c933d89a49de as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/hbase/meta/1588230740/ns/76c9eed7a2024110a3e9c933d89a49de 2024-11-12T14:19:45,644 INFO [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/hbase/meta/1588230740/ns/76c9eed7a2024110a3e9c933d89a49de, entries=8, sequenceid=90, filesize=5.8 K 2024-11-12T14:19:45,645 DEBUG [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/hbase/meta/1588230740/.tmp/rep_barrier/0ba964ec5e5942c4bbb0252a3012363f as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/hbase/meta/1588230740/rep_barrier/0ba964ec5e5942c4bbb0252a3012363f 2024-11-12T14:19:45,647 INFO [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/hbase/meta/1588230740/rep_barrier/0ba964ec5e5942c4bbb0252a3012363f, entries=6, sequenceid=90, filesize=5.9 K 2024-11-12T14:19:45,649 DEBUG [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/hbase/meta/1588230740/.tmp/table/d5d35076ec0e45feb567504c25b6ef1b as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/hbase/meta/1588230740/table/d5d35076ec0e45feb567504c25b6ef1b 2024-11-12T14:19:45,652 INFO [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/hbase/meta/1588230740/table/d5d35076ec0e45feb567504c25b6ef1b, entries=7, sequenceid=90, filesize=5.8 K 2024-11-12T14:19:45,652 INFO [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1698ms, sequenceid=90, compaction requested=false 2024-11-12T14:19:45,657 DEBUG [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-12T14:19:45,657 DEBUG [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-12T14:19:45,658 INFO [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-12T14:19:45,658 DEBUG [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731421183953Running coprocessor pre-close hooks at 1731421183953Disabling compacts and flushes for region at 1731421183953Disabling writes for close at 1731421183954 (+1 ms)Obtaining lock to block concurrent updates at 1731421183954Preparing flush snapshotting stores in 1588230740 at 1731421183954Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731421183954Flushing stores of hbase:meta,,1.1588230740 at 1731421183955 (+1 ms)Flushing 1588230740/info: creating writer at 1731421183955Flushing 1588230740/info: appending metadata at 1731421183973 (+18 ms)Flushing 1588230740/info: closing flushed file at 1731421183973Flushing 1588230740/ns: creating writer at 1731421184380 (+407 ms)Flushing 1588230740/ns: appending metadata at 1731421184393 (+13 ms)Flushing 1588230740/ns: closing flushed file at 1731421184393Flushing 1588230740/rep_barrier: creating writer at 1731421184799 (+406 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731421184812 (+13 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731421184812Flushing 1588230740/table: creating writer at 1731421185219 (+407 ms)Flushing 1588230740/table: appending metadata at 1731421185232 (+13 ms)Flushing 1588230740/table: closing flushed file at 1731421185232Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73eaa977: reopening flushed file at 1731421185638 (+406 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5a892ed4: reopening flushed file at 1731421185641 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c8697fc: reopening flushed file at 1731421185645 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@15a0921d: reopening flushed file at 1731421185648 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1698ms, sequenceid=90, compaction requested=false at 1731421185652 (+4 ms)Writing region close event to WAL at 1731421185654 (+2 ms)Running coprocessor post-close hooks at 1731421185657 (+3 ms)Closed at 1731421185657 2024-11-12T14:19:45,658 DEBUG [RS_CLOSE_META-regionserver/2b6d221c5cde:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-12T14:19:45,755 INFO [RS:0;2b6d221c5cde:42925 {}] regionserver.HRegionServer(976): stopping server 2b6d221c5cde,42925,1731420990822; all regions closed. 2024-11-12T14:19:45,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741834_1010 (size=25344) 2024-11-12T14:19:45,762 DEBUG [RS:0;2b6d221c5cde:42925 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/oldWALs 2024-11-12T14:19:45,762 INFO [RS:0;2b6d221c5cde:42925 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2b6d221c5cde%2C42925%2C1731420990822.meta:.meta(num 1731420995596) 2024-11-12T14:19:45,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741833_1009 (size=18859327) 2024-11-12T14:19:45,765 DEBUG [RS:0;2b6d221c5cde:42925 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/oldWALs 2024-11-12T14:19:45,765 INFO [RS:0;2b6d221c5cde:42925 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 2b6d221c5cde%2C42925%2C1731420990822:(num 1731420995070) 2024-11-12T14:19:45,765 DEBUG [RS:0;2b6d221c5cde:42925 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-12T14:19:45,765 INFO [RS:0;2b6d221c5cde:42925 {}] regionserver.LeaseManager(133): Closed leases 2024-11-12T14:19:45,765 INFO [RS:0;2b6d221c5cde:42925 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T14:19:45,765 INFO [RS:0;2b6d221c5cde:42925 {}] hbase.ChoreService(370): Chore service for: regionserver/2b6d221c5cde:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-12T14:19:45,765 INFO [RS:0;2b6d221c5cde:42925 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T14:19:45,765 INFO [regionserver/2b6d221c5cde:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T14:19:45,766 INFO [RS:0;2b6d221c5cde:42925 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:42925 2024-11-12T14:19:45,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45591-0x1012f67a1e60000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-12T14:19:45,817 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42925-0x1012f67a1e60001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/2b6d221c5cde,42925,1731420990822 2024-11-12T14:19:45,817 INFO [RS:0;2b6d221c5cde:42925 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T14:19:45,818 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [2b6d221c5cde,42925,1731420990822] 2024-11-12T14:19:45,901 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/2b6d221c5cde,42925,1731420990822 already deleted, retry=false 2024-11-12T14:19:45,901 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 2b6d221c5cde,42925,1731420990822 expired; onlineServers=0 2024-11-12T14:19:45,902 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '2b6d221c5cde,45591,1731420989939' ***** 2024-11-12T14:19:45,902 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-12T14:19:45,902 INFO [M:0;2b6d221c5cde:45591 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-12T14:19:45,902 INFO [M:0;2b6d221c5cde:45591 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-12T14:19:45,902 DEBUG [M:0;2b6d221c5cde:45591 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-12T14:19:45,902 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-12T14:19:45,902 DEBUG [M:0;2b6d221c5cde:45591 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-12T14:19:45,902 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster-HFileCleaner.large.0-1731420994662 {}] cleaner.HFileCleaner(306): Exit Thread[master/2b6d221c5cde:0:becomeActiveMaster-HFileCleaner.large.0-1731420994662,5,FailOnTimeoutGroup] 2024-11-12T14:19:45,902 INFO [M:0;2b6d221c5cde:45591 {}] hbase.ChoreService(370): Chore service for: master/2b6d221c5cde:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-12T14:19:45,902 INFO [M:0;2b6d221c5cde:45591 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-12T14:19:45,902 DEBUG [master/2b6d221c5cde:0:becomeActiveMaster-HFileCleaner.small.0-1731420994669 {}] cleaner.HFileCleaner(306): Exit Thread[master/2b6d221c5cde:0:becomeActiveMaster-HFileCleaner.small.0-1731420994669,5,FailOnTimeoutGroup] 2024-11-12T14:19:45,903 DEBUG [M:0;2b6d221c5cde:45591 {}] master.HMaster(1795): Stopping service threads 2024-11-12T14:19:45,903 INFO [M:0;2b6d221c5cde:45591 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-12T14:19:45,903 INFO [M:0;2b6d221c5cde:45591 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-12T14:19:45,903 ERROR [M:0;2b6d221c5cde:45591 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[IPC Client (1839957908) connection to localhost/127.0.0.1:34349 from jenkins,5,PEWorkerGroup] Thread[IPC Parameter Sending Thread for localhost/127.0.0.1:34349,5,PEWorkerGroup] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-12T14:19:45,903 INFO [M:0;2b6d221c5cde:45591 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-12T14:19:45,904 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-12T14:19:45,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45591-0x1012f67a1e60000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-12T14:19:45,965 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45591-0x1012f67a1e60000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-12T14:19:45,965 DEBUG [M:0;2b6d221c5cde:45591 {}] zookeeper.ZKUtil(347): master:45591-0x1012f67a1e60000, quorum=127.0.0.1:50547, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-12T14:19:45,965 WARN [M:0;2b6d221c5cde:45591 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-12T14:19:45,966 INFO [M:0;2b6d221c5cde:45591 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/.lastflushedseqids 2024-11-12T14:19:45,976 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742528_1704 (size=72) 2024-11-12T14:19:46,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42925-0x1012f67a1e60001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:19:46,003 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42925-0x1012f67a1e60001, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:19:46,003 INFO [RS:0;2b6d221c5cde:42925 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T14:19:46,003 INFO [RS:0;2b6d221c5cde:42925 {}] regionserver.HRegionServer(1031): Exiting; stopping=2b6d221c5cde,42925,1731420990822; zookeeper connection closed. 2024-11-12T14:19:46,003 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@17824dc9 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@17824dc9 2024-11-12T14:19:46,004 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-12T14:19:46,376 INFO [M:0;2b6d221c5cde:45591 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-12T14:19:46,377 INFO [M:0;2b6d221c5cde:45591 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-12T14:19:46,377 DEBUG [M:0;2b6d221c5cde:45591 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-12T14:19:46,377 INFO [M:0;2b6d221c5cde:45591 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:19:46,377 DEBUG [M:0;2b6d221c5cde:45591 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:19:46,377 DEBUG [M:0;2b6d221c5cde:45591 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-12T14:19:46,377 DEBUG [M:0;2b6d221c5cde:45591 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:19:46,377 INFO [M:0;2b6d221c5cde:45591 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=801.21 KB heapSize=984.08 KB 2024-11-12T14:19:46,391 DEBUG [M:0;2b6d221c5cde:45591 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/45bad824a7d24eea8559e17d2c1091d5 is 82, key is hbase:meta,,1/info:regioninfo/1731420995710/Put/seqid=0 2024-11-12T14:19:46,397 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742529_1705 (size=5672) 2024-11-12T14:19:46,397 INFO [M:0;2b6d221c5cde:45591 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2271 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/45bad824a7d24eea8559e17d2c1091d5 2024-11-12T14:19:46,418 DEBUG [M:0;2b6d221c5cde:45591 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/edbdc8b4fc2e4bd2bae283ef1f11dae4 is 2480, key is \x00\x00\x00\x00\x00\x00\x00\x8B/proc:d/1731421152251/Put/seqid=0 2024-11-12T14:19:46,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742530_1706 (size=44645) 2024-11-12T14:19:46,821 INFO [M:0;2b6d221c5cde:45591 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=800.66 KB at sequenceid=2271 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/edbdc8b4fc2e4bd2bae283ef1f11dae4 2024-11-12T14:19:46,825 INFO [M:0;2b6d221c5cde:45591 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for edbdc8b4fc2e4bd2bae283ef1f11dae4 2024-11-12T14:19:46,839 DEBUG [M:0;2b6d221c5cde:45591 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/31d3e7892e354eaca7c438d85b479c6e is 69, key is 2b6d221c5cde,42925,1731420990822/rs:state/1731420994745/Put/seqid=0 2024-11-12T14:19:46,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073742531_1707 (size=5156) 2024-11-12T14:19:46,846 INFO [M:0;2b6d221c5cde:45591 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2271 (bloomFilter=true), to=hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/31d3e7892e354eaca7c438d85b479c6e 2024-11-12T14:19:46,850 DEBUG [M:0;2b6d221c5cde:45591 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/45bad824a7d24eea8559e17d2c1091d5 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/45bad824a7d24eea8559e17d2c1091d5 2024-11-12T14:19:46,852 INFO [M:0;2b6d221c5cde:45591 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/45bad824a7d24eea8559e17d2c1091d5, entries=8, sequenceid=2271, filesize=5.5 K 2024-11-12T14:19:46,853 DEBUG [M:0;2b6d221c5cde:45591 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/edbdc8b4fc2e4bd2bae283ef1f11dae4 as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/edbdc8b4fc2e4bd2bae283ef1f11dae4 2024-11-12T14:19:46,855 INFO [M:0;2b6d221c5cde:45591 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for edbdc8b4fc2e4bd2bae283ef1f11dae4 2024-11-12T14:19:46,855 INFO [M:0;2b6d221c5cde:45591 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/edbdc8b4fc2e4bd2bae283ef1f11dae4, entries=166, sequenceid=2271, filesize=43.6 K 2024-11-12T14:19:46,856 DEBUG [M:0;2b6d221c5cde:45591 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/31d3e7892e354eaca7c438d85b479c6e as hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/31d3e7892e354eaca7c438d85b479c6e 2024-11-12T14:19:46,858 INFO [M:0;2b6d221c5cde:45591 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34349/user/jenkins/test-data/de6dd569-41e3-6582-bbe1-a8b3a9ed80d3/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/31d3e7892e354eaca7c438d85b479c6e, entries=1, sequenceid=2271, filesize=5.0 K 2024-11-12T14:19:46,859 INFO [M:0;2b6d221c5cde:45591 {}] regionserver.HRegion(3140): Finished flush of dataSize ~801.21 KB/820440, heapSize ~983.78 KB/1007392, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 482ms, sequenceid=2271, compaction requested=false 2024-11-12T14:19:46,860 INFO [M:0;2b6d221c5cde:45591 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-12T14:19:46,860 DEBUG [M:0;2b6d221c5cde:45591 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731421186377Disabling compacts and flushes for region at 1731421186377Disabling writes for close at 1731421186377Obtaining lock to block concurrent updates at 1731421186377Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731421186377Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=820440, getHeapSize=1007632, getOffHeapSize=0, getCellsCount=2462 at 1731421186377Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731421186378 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731421186378Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731421186391 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731421186391Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731421186400 (+9 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731421186418 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731421186418Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731421186825 (+407 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731421186839 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731421186839Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@334c5f37: reopening flushed file at 1731421186849 (+10 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@711c0973: reopening flushed file at 1731421186852 (+3 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4539ed09: reopening flushed file at 1731421186855 (+3 ms)Finished flush of dataSize ~801.21 KB/820440, heapSize ~983.78 KB/1007392, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 482ms, sequenceid=2271, compaction requested=false at 1731421186859 (+4 ms)Writing region close event to WAL at 1731421186860 (+1 ms)Closed at 1731421186860 2024-11-12T14:19:46,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:42533 is added to blk_1073741830_1006 (size=969238) 2024-11-12T14:19:46,864 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-12T14:19:46,864 INFO [M:0;2b6d221c5cde:45591 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-12T14:19:46,864 INFO [M:0;2b6d221c5cde:45591 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:45591 2024-11-12T14:19:46,864 INFO [M:0;2b6d221c5cde:45591 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-12T14:19:46,998 INFO [M:0;2b6d221c5cde:45591 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-12T14:19:46,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45591-0x1012f67a1e60000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:19:46,998 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:45591-0x1012f67a1e60000, quorum=127.0.0.1:50547, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-12T14:19:47,014 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@56f5dfd8{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-12T14:19:47,017 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@5f3319b1{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T14:19:47,017 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T14:19:47,018 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@288b0180{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T14:19:47,018 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@183dc930{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/hadoop.log.dir/,STOPPED} 2024-11-12T14:19:47,021 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-12T14:19:47,021 WARN [BP-1636810488-172.17.0.3-1731420984850 heartbeating to localhost/127.0.0.1:34349 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-12T14:19:47,021 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-12T14:19:47,021 WARN [BP-1636810488-172.17.0.3-1731420984850 heartbeating to localhost/127.0.0.1:34349 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1636810488-172.17.0.3-1731420984850 (Datanode Uuid 81b1dfbb-fc6a-4a67-8a95-69c2120de4e8) service to localhost/127.0.0.1:34349 2024-11-12T14:19:47,024 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/cluster_a8bb44ce-6e49-fcce-04c9-975e63e4e40e/data/data1/current/BP-1636810488-172.17.0.3-1731420984850 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T14:19:47,026 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-12T14:19:47,027 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/cluster_a8bb44ce-6e49-fcce-04c9-975e63e4e40e/data/data2/current/BP-1636810488-172.17.0.3-1731420984850 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-12T14:19:47,037 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@322e478f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-12T14:19:47,041 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@4b008a6a{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-12T14:19:47,041 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-12T14:19:47,041 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@d942d39{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-12T14:19:47,041 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@71c9bba7{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_branch-3/hbase-server/target/test-data/1caa6b08-b6b1-e875-09c3-293f469da158/hadoop.log.dir/,STOPPED} 2024-11-12T14:19:47,062 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-12T14:19:47,236 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down